diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index 46bdf208be6ad..36c42fced93d0 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -165,6 +165,12 @@ bool shouldPrint(const BinaryFunction &Function) {
     }
   }
 
+  std::optional<StringRef> Origin = Function.getOriginSectionName();
+  if (Origin && llvm::any_of(opts::PrintOnly, [&](const std::string &Name) {
+        return Name == *Origin;
+      }))
+    return true;
+
   return false;
 }
 
diff --git a/bolt/test/AArch64/constant_island_pie_update.s b/bolt/test/AArch64/constant_island_pie_update.s
index 313e103b19c05..889f6b6acd9f2 100644
--- a/bolt/test/AArch64/constant_island_pie_update.s
+++ b/bolt/test/AArch64/constant_island_pie_update.s
@@ -8,15 +8,15 @@
 # RUN: %clang %cflags -fPIC -pie %t.o -o %t.rela.exe -nostdlib \
 # RUN:   -Wl,-q -Wl,-z,notext
 # RUN: llvm-bolt %t.rela.exe -o %t.rela.bolt --use-old-text=0 --lite=0
-# RUN: llvm-objdump -j .text -d --show-all-symbols %t.rela.bolt | FileCheck %s
+# RUN: llvm-objdump -j .text -d -z --show-all-symbols %t.rela.bolt | FileCheck %s
 # RUN: llvm-readelf -rsW %t.rela.bolt | FileCheck --check-prefix=ELFCHECK %s
 // .relr.dyn
 # RUN: %clang %cflags -fPIC -pie %t.o -o %t.relr.exe -nostdlib \
 # RUN:   -Wl,-q -Wl,-z,notext -Wl,--pack-dyn-relocs=relr
 # RUN: llvm-objcopy --remove-section .rela.mytext %t.relr.exe
 # RUN: llvm-bolt %t.relr.exe -o %t.relr.bolt --use-old-text=0 --lite=0
-# RUN: llvm-objdump -j .text -d --show-all-symbols %t.relr.bolt | FileCheck %s
-# RUN: llvm-objdump -j .text -d %t.relr.bolt | \
+# RUN: llvm-objdump -j .text -d -z --show-all-symbols %t.relr.bolt | FileCheck %s
+# RUN: llvm-objdump -j .text -d -z %t.relr.bolt | \
 # RUN:   FileCheck %s --check-prefix=ADDENDCHECK
 # RUN: llvm-readelf -rsW %t.relr.bolt | FileCheck --check-prefix=RELRELFCHECK %s
 # RUN: llvm-readelf -SW %t.relr.bolt | FileCheck --check-prefix=RELRSZCHECK %s
diff --git a/bolt/test/AArch64/update-weak-reference-symbol.s b/bolt/test/AArch64/update-weak-reference-symbol.s
index 600a06b8b6d8f..46819e888b08e 100644
--- a/bolt/test/AArch64/update-weak-reference-symbol.s
+++ b/bolt/test/AArch64/update-weak-reference-symbol.s
@@ -3,7 +3,7 @@
 // RUN: %clang %cflags -Wl,-z,notext -shared -Wl,-q %s -o %t.so
 // RUN: llvm-bolt %t.so -o %t.so.bolt
 // RUN: llvm-nm -n %t.so.bolt > %t.out.txt
-// RUN: llvm-objdump -dj .rodata %t.so.bolt >> %t.out.txt
+// RUN: llvm-objdump -z -dj .rodata %t.so.bolt >> %t.out.txt
 // RUN: FileCheck %s --input-file=%t.out.txt
 
 # CHECK: w func_1
diff --git a/bolt/test/X86/print-only-section.s b/bolt/test/X86/print-only-section.s
new file mode 100644
index 0000000000000..d580818ca4fc6
--- /dev/null
+++ b/bolt/test/X86/print-only-section.s
@@ -0,0 +1,29 @@
+## Check that --print-only flag works with sections.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t.exe
+# RUN: llvm-bolt %t.exe -o %t.out --print-cfg --print-only=unused_code 2>&1 \
+# RUN:   | FileCheck %s
+
+# CHECK: Binary Function "foo"
+# CHECK-NOT: Binary Function "_start"
+
+  .text
+  .globl _start
+  .type _start, %function
+_start:
+  .cfi_startproc
+  ret
+  .cfi_endproc
+  .size _start, .-_start
+
+  .section unused_code,"ax",@progbits
+  .globl foo
+  .type foo, %function
+foo:
+  .cfi_startproc
+  ret
+  .cfi_endproc
+  .size foo, .-foo
diff --git a/bolt/test/merge-fdata-uninitialized-header.test b/bolt/test/merge-fdata-uninitialized-header.test
new file mode 100644
index 0000000000000..5336961278411
--- /dev/null
+++ b/bolt/test/merge-fdata-uninitialized-header.test
@@ -0,0 +1,45 @@
+## Test that merge-fdata correctly handles YAML header with an uninitialized
+## fields. a.yaml does not have hash-func set and it used to crash merge-fdata.
+
+# REQUIRES: system-linux
+
+# RUN: split-file %s %t
+# RUN: not merge-fdata %t/a.yaml %t/b.yaml 2>&1 | FileCheck %s
+
+# CHECK: cannot merge profiles with different hash functions
+
+#--- a.yaml
+---
+header:
+  profile-version: 1
+  binary-name:     'a.out'
+  binary-build-id: '<unknown>'
+  profile-flags:   [ lbr ]
+  profile-origin:  branch profile reader
+  profile-events:  ''
+  dfs-order:       false
+functions:
+  - name:            'main'
+    fid:             1
+    hash:            0x50BBA3441D436491
+    exec:            1
+    nblocks:         0
+...
+#--- b.yaml
+---
+header:
+  profile-version: 1
+  binary-name:     'a.out'
+  binary-build-id: '<unknown>'
+  profile-flags:   [ lbr ]
+  profile-origin:  branch profile reader
+  profile-events:  ''
+  dfs-order:       false
+  hash-func: xxh3
+functions:
+  - name:            'main'
+    fid:             1
+    hash:            0x50BBA3441D436491
+    exec:            1
+    nblocks:         0
+...
diff --git a/bolt/tools/merge-fdata/merge-fdata.cpp b/bolt/tools/merge-fdata/merge-fdata.cpp
index b640aae808f56..89ca46c1c0a8f 100644
--- a/bolt/tools/merge-fdata/merge-fdata.cpp
+++ b/bolt/tools/merge-fdata/merge-fdata.cpp
@@ -145,6 +145,10 @@ void mergeProfileHeaders(BinaryProfileHeader &MergedHeader,
     errs() << "WARNING: merging profiles with different sampling events\n";
     MergedHeader.EventNames += "," + Header.EventNames;
   }
+
+  if (MergedHeader.HashFunction != Header.HashFunction)
+    report_error("merge conflict",
+                 "cannot merge profiles with different hash functions");
 }
 
 void mergeBasicBlockProfile(BinaryBasicBlockProfile &MergedBB,
@@ -386,6 +390,7 @@ int main(int argc, char **argv) {
   // Merged information for all functions.
   StringMap<BinaryFunctionProfile> MergedBFs;
 
+  bool FirstHeader = true;
   for (std::string &InputDataFilename : Inputs) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
         MemoryBuffer::getFileOrSTDIN(InputDataFilename);
@@ -409,7 +414,12 @@ int main(int argc, char **argv) {
     }
 
     // Merge the header.
-    mergeProfileHeaders(MergedHeader, BP.Header);
+    if (FirstHeader) {
+      MergedHeader = BP.Header;
+      FirstHeader = false;
+    } else {
+      mergeProfileHeaders(MergedHeader, BP.Header);
+    }
 
     // Do the function merge.
     for (BinaryFunctionProfile &BF : BP.Functions) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
index 378427a1eab00..f05924b81c4c0 100644
--- a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
@@ -7,19 +7,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "PosixReturnCheck.h"
-#include "../utils/Matchers.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Lexer.h"
 
 using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
-static StringRef getFunctionSpelling(const MatchFinder::MatchResult &Result,
-                                     const char *BindingStr) {
-  const CallExpr *MatchedCall = cast<CallExpr>(
-      (Result.Nodes.getNodeAs<BinaryOperator>(BindingStr))->getLHS());
+static StringRef getFunctionSpelling(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedCall = Result.Nodes.getNodeAs<CallExpr>("call");
   const SourceManager &SM = *Result.SourceManager;
   return Lexer::getSourceText(CharSourceRange::getTokenRange(
                                   MatchedCall->getCallee()->getSourceRange()),
@@ -27,32 +25,40 @@ static StringRef getFunctionSpelling(const MatchFinder::MatchResult &Result,
 }
 
 void PosixReturnCheck::registerMatchers(MatchFinder *Finder) {
+  const auto PosixCall =
+      callExpr(callee(functionDecl(
+                   anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
+                   unless(hasName("::posix_openpt")))))
+          .bind("call");
+  const auto ZeroIntegerLiteral = integerLiteral(equals(0));
+  const auto NegIntegerLiteral =
+      unaryOperator(hasOperatorName("-"), hasUnaryOperand(integerLiteral()));
+
   Finder->addMatcher(
       binaryOperator(
-          hasOperatorName("<"),
-          hasLHS(callExpr(callee(functionDecl(
-              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
-              unless(hasName("::posix_openpt")))))),
-          hasRHS(integerLiteral(equals(0))))
+          anyOf(allOf(hasOperatorName("<"), hasLHS(PosixCall),
+                      hasRHS(ZeroIntegerLiteral)),
+                allOf(hasOperatorName(">"), hasLHS(ZeroIntegerLiteral),
+                      hasRHS(PosixCall))))
           .bind("ltzop"),
       this);
   Finder->addMatcher(
       binaryOperator(
-          hasOperatorName(">="),
-          hasLHS(callExpr(callee(functionDecl(
-              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
-              unless(hasName("::posix_openpt")))))),
-          hasRHS(integerLiteral(equals(0))))
+          anyOf(allOf(hasOperatorName(">="), hasLHS(PosixCall),
+                      hasRHS(ZeroIntegerLiteral)),
+                allOf(hasOperatorName("<="), hasLHS(ZeroIntegerLiteral),
+                      hasRHS(PosixCall))))
           .bind("atop"),
       this);
+  Finder->addMatcher(binaryOperator(hasAnyOperatorName("==", "!="),
+                                    hasOperands(PosixCall, NegIntegerLiteral))
+                         .bind("binop"),
+                     this);
   Finder->addMatcher(
-      binaryOperator(
-          hasAnyOperatorName("==", "!=", "<=", "<"),
-          hasLHS(callExpr(callee(functionDecl(
-              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
-              unless(hasName("::posix_openpt")))))),
-          hasRHS(unaryOperator(hasOperatorName("-"),
-                               hasUnaryOperand(integerLiteral()))))
+      binaryOperator(anyOf(allOf(hasAnyOperatorName("<=", "<"),
+                                 hasLHS(PosixCall), hasRHS(NegIntegerLiteral)),
+                           allOf(hasAnyOperatorName(">", ">="),
+                                 hasLHS(NegIntegerLiteral), hasRHS(PosixCall))))
           .bind("binop"),
       this);
 }
@@ -61,10 +67,13 @@ void PosixReturnCheck::check(const MatchFinder::MatchResult &Result) {
   if (const auto *LessThanZeroOp =
           Result.Nodes.getNodeAs<BinaryOperator>("ltzop")) {
     SourceLocation OperatorLoc = LessThanZeroOp->getOperatorLoc();
+    StringRef NewBinOp =
+        LessThanZeroOp->getOpcode() == BinaryOperator::Opcode::BO_LT ? ">"
+                                                                     : "<";
     diag(OperatorLoc, "the comparison always evaluates to false because %0 "
                       "always returns non-negative values")
-        << getFunctionSpelling(Result, "ltzop")
-        << FixItHint::CreateReplacement(OperatorLoc, Twine(">").str());
+        << getFunctionSpelling(Result)
+        << FixItHint::CreateReplacement(OperatorLoc, NewBinOp);
     return;
   }
   if (const auto *AlwaysTrueOp =
@@ -72,12 +81,12 @@ void PosixReturnCheck::check(const MatchFinder::MatchResult &Result) {
     diag(AlwaysTrueOp->getOperatorLoc(),
          "the comparison always evaluates to true because %0 always returns "
          "non-negative values")
-        << getFunctionSpelling(Result, "atop");
+        << getFunctionSpelling(Result);
     return;
   }
   const auto *BinOp = Result.Nodes.getNodeAs<BinaryOperator>("binop");
   diag(BinOp->getOperatorLoc(), "%0 only returns non-negative values")
-      << getFunctionSpelling(Result, "binop");
+      << getFunctionSpelling(Result);
 }
 
 } // namespace clang::tidy::bugprone
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
index ea7eaa0b0ff81..604a7cac0e490 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "UnsafeFunctionsCheck.h"
+#include "../utils/OptionsUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/PPCallbacks.h"
@@ -18,6 +19,10 @@ using namespace llvm;
 
 namespace clang::tidy::bugprone {
 
+static constexpr llvm::StringLiteral OptionNameCustomFunctions =
+    "CustomFunctions";
+static constexpr llvm::StringLiteral OptionNameReportDefaultFunctions =
+    "ReportDefaultFunctions";
 static constexpr llvm::StringLiteral OptionNameReportMoreUnsafeFunctions =
     "ReportMoreUnsafeFunctions";
 
@@ -26,6 +31,8 @@ static constexpr llvm::StringLiteral FunctionNamesWithAnnexKReplacementId =
 static constexpr llvm::StringLiteral FunctionNamesId = "FunctionsNames";
 static constexpr llvm::StringLiteral AdditionalFunctionNamesId =
     "AdditionalFunctionsNames";
+static constexpr llvm::StringLiteral CustomFunctionNamesId =
+    "CustomFunctionNames";
 static constexpr llvm::StringLiteral DeclRefId = "DRE";
 
 static std::optional<std::string>
@@ -127,57 +134,128 @@ static bool isAnnexKAvailable(std::optional<bool> &CacheVar, Preprocessor *PP,
   return CacheVar.value();
 }
 
+static std::vector<UnsafeFunctionsCheck::CheckedFunction>
+parseCheckedFunctions(StringRef Option, ClangTidyContext *Context) {
+  const std::vector<StringRef> Functions =
+      utils::options::parseStringList(Option);
+  std::vector<UnsafeFunctionsCheck::CheckedFunction> Result;
+  Result.reserve(Functions.size());
+
+  for (StringRef Function : Functions) {
+    if (Function.empty())
+      continue;
+
+    const auto [Name, Rest] = Function.split(',');
+    const auto [Replacement, Reason] = Rest.split(',');
+
+    if (Name.trim().empty()) {
+      Context->configurationDiag("invalid configuration value for option '%0'; "
+                                 "expected the name of an unsafe function")
+          << OptionNameCustomFunctions;
+      continue;
+    }
+
+    Result.push_back(
+        {Name.trim().str(),
+         matchers::MatchesAnyListedNameMatcher::NameMatcher(Name.trim()),
+         Replacement.trim().str(), Reason.trim().str()});
+  }
+
+  return Result;
+}
+
+static std::string serializeCheckedFunctions(
+    const std::vector<UnsafeFunctionsCheck::CheckedFunction> &Functions) {
+  std::vector<std::string> Result;
+  Result.reserve(Functions.size());
+
+  for (const auto &Entry : Functions) {
+    if (Entry.Reason.empty())
+      Result.push_back(Entry.Name + "," + Entry.Replacement);
+    else
+      Result.push_back(Entry.Name + "," + Entry.Replacement + "," +
+                       Entry.Reason);
+  }
+
+  return llvm::join(Result, ";");
+}
+
 UnsafeFunctionsCheck::UnsafeFunctionsCheck(StringRef Name,
                                            ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
+      CustomFunctions(parseCheckedFunctions(
+          Options.get(OptionNameCustomFunctions, ""), Context)),
+      ReportDefaultFunctions(
+          Options.get(OptionNameReportDefaultFunctions, true)),
       ReportMoreUnsafeFunctions(
           Options.get(OptionNameReportMoreUnsafeFunctions, true)) {}
 
 void UnsafeFunctionsCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, OptionNameCustomFunctions,
+                serializeCheckedFunctions(CustomFunctions));
+  Options.store(Opts, OptionNameReportDefaultFunctions, ReportDefaultFunctions);
   Options.store(Opts, OptionNameReportMoreUnsafeFunctions,
                 ReportMoreUnsafeFunctions);
 }
 
 void UnsafeFunctionsCheck::registerMatchers(MatchFinder *Finder) {
-  if (getLangOpts().C11) {
-    // Matching functions with safe replacements only in Annex K.
-    auto FunctionNamesWithAnnexKReplacementMatcher = hasAnyName(
-        "::bsearch", "::ctime", "::fopen", "::fprintf", "::freopen", "::fscanf",
-        "::fwprintf", "::fwscanf", "::getenv", "::gmtime", "::localtime",
-        "::mbsrtowcs", "::mbstowcs", "::memcpy", "::memmove", "::memset",
-        "::printf", "::qsort", "::scanf", "::snprintf", "::sprintf", "::sscanf",
-        "::strcat", "::strcpy", "::strerror", "::strlen", "::strncat",
-        "::strncpy", "::strtok", "::swprintf", "::swscanf", "::vfprintf",
-        "::vfscanf", "::vfwprintf", "::vfwscanf", "::vprintf", "::vscanf",
-        "::vsnprintf", "::vsprintf", "::vsscanf", "::vswprintf", "::vswscanf",
-        "::vwprintf", "::vwscanf", "::wcrtomb", "::wcscat", "::wcscpy",
-        "::wcslen", "::wcsncat", "::wcsncpy", "::wcsrtombs", "::wcstok",
-        "::wcstombs", "::wctomb", "::wmemcpy", "::wmemmove", "::wprintf",
-        "::wscanf");
+  if (ReportDefaultFunctions) {
+    if (getLangOpts().C11) {
+      // Matching functions with safe replacements only in Annex K.
+      auto FunctionNamesWithAnnexKReplacementMatcher = hasAnyName(
+          "::bsearch", "::ctime", "::fopen", "::fprintf", "::freopen",
+          "::fscanf", "::fwprintf", "::fwscanf", "::getenv", "::gmtime",
+          "::localtime", "::mbsrtowcs", "::mbstowcs", "::memcpy", "::memmove",
+          "::memset", "::printf", "::qsort", "::scanf", "::snprintf",
+          "::sprintf", "::sscanf", "::strcat", "::strcpy", "::strerror",
+          "::strlen", "::strncat", "::strncpy", "::strtok", "::swprintf",
+          "::swscanf", "::vfprintf", "::vfscanf", "::vfwprintf", "::vfwscanf",
+          "::vprintf", "::vscanf", "::vsnprintf", "::vsprintf", "::vsscanf",
+          "::vswprintf", "::vswscanf", "::vwprintf", "::vwscanf", "::wcrtomb",
+          "::wcscat", "::wcscpy", "::wcslen", "::wcsncat", "::wcsncpy",
+          "::wcsrtombs", "::wcstok", "::wcstombs", "::wctomb", "::wmemcpy",
+          "::wmemmove", "::wprintf", "::wscanf");
+      Finder->addMatcher(
+          declRefExpr(to(functionDecl(FunctionNamesWithAnnexKReplacementMatcher)
+                             .bind(FunctionNamesWithAnnexKReplacementId)))
+              .bind(DeclRefId),
+          this);
+    }
+
+    // Matching functions with replacements without Annex K.
+    auto FunctionNamesMatcher =
+        hasAnyName("::asctime", "asctime_r", "::gets", "::rewind", "::setbuf");
     Finder->addMatcher(
-        declRefExpr(to(functionDecl(FunctionNamesWithAnnexKReplacementMatcher)
-                           .bind(FunctionNamesWithAnnexKReplacementId)))
+        declRefExpr(
+            to(functionDecl(FunctionNamesMatcher).bind(FunctionNamesId)))
             .bind(DeclRefId),
         this);
+
+    if (ReportMoreUnsafeFunctions) {
+      // Matching functions with replacements without Annex K, at user request.
+      auto AdditionalFunctionNamesMatcher =
+          hasAnyName("::bcmp", "::bcopy", "::bzero", "::getpw", "::vfork");
+      Finder->addMatcher(
+          declRefExpr(to(functionDecl(AdditionalFunctionNamesMatcher)
+                             .bind(AdditionalFunctionNamesId)))
+              .bind(DeclRefId),
+          this);
+    }
   }
 
-  // Matching functions with replacements without Annex K.
-  auto FunctionNamesMatcher =
-      hasAnyName("::asctime", "asctime_r", "::gets", "::rewind", "::setbuf");
-  Finder->addMatcher(
-      declRefExpr(to(functionDecl(FunctionNamesMatcher).bind(FunctionNamesId)))
-          .bind(DeclRefId),
-      this);
-
-  if (ReportMoreUnsafeFunctions) {
-    // Matching functions with replacements without Annex K, at user request.
-    auto AdditionalFunctionNamesMatcher =
-        hasAnyName("::bcmp", "::bcopy", "::bzero", "::getpw", "::vfork");
-    Finder->addMatcher(
-        declRefExpr(to(functionDecl(AdditionalFunctionNamesMatcher)
-                           .bind(AdditionalFunctionNamesId)))
-            .bind(DeclRefId),
-        this);
+  if (!CustomFunctions.empty()) {
+    std::vector<llvm::StringRef> FunctionNames;
+    FunctionNames.reserve(CustomFunctions.size());
+
+    for (const auto &Entry : CustomFunctions)
+      FunctionNames.push_back(Entry.Name);
+
+    auto CustomFunctionsMatcher = matchers::matchesAnyListedName(FunctionNames);
+
+    Finder->addMatcher(declRefExpr(to(functionDecl(CustomFunctionsMatcher)
+                                          .bind(CustomFunctionNamesId)))
+                           .bind(DeclRefId),
+                       this);
   }
 }
 
@@ -186,16 +264,46 @@ void UnsafeFunctionsCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *FuncDecl = cast<FunctionDecl>(DeclRef->getDecl());
   assert(DeclRef && FuncDecl && "No valid matched node in check()");
 
+  // Only one of these are matched at a time.
   const auto *AnnexK = Result.Nodes.getNodeAs<FunctionDecl>(
       FunctionNamesWithAnnexKReplacementId);
   const auto *Normal = Result.Nodes.getNodeAs<FunctionDecl>(FunctionNamesId);
   const auto *Additional =
       Result.Nodes.getNodeAs<FunctionDecl>(AdditionalFunctionNamesId);
-  assert((AnnexK || Normal || Additional) && "No valid match category.");
+  const auto *Custom =
+      Result.Nodes.getNodeAs<FunctionDecl>(CustomFunctionNamesId);
+  assert((AnnexK || Normal || Additional || Custom) &&
+         "No valid match category.");
 
   bool AnnexKIsAvailable =
       isAnnexKAvailable(IsAnnexKAvailable, PP, getLangOpts());
   StringRef FunctionName = FuncDecl->getName();
+
+  if (Custom) {
+    for (const auto &Entry : CustomFunctions) {
+      if (Entry.Pattern.match(*FuncDecl)) {
+        StringRef Reason =
+            Entry.Reason.empty() ? "is marked as unsafe" : Entry.Reason.c_str();
+
+        if (Entry.Replacement.empty()) {
+          diag(DeclRef->getExprLoc(), "function %0 %1; it should not be used")
+              << FuncDecl << Reason << Entry.Replacement
+              << DeclRef->getSourceRange();
+        } else {
+          diag(DeclRef->getExprLoc(),
+               "function %0 %1; '%2' should be used instead")
+              << FuncDecl << Reason << Entry.Replacement
+              << DeclRef->getSourceRange();
+        }
+
+        return;
+      }
+    }
+
+    llvm_unreachable("No custom function was matched.");
+    return;
+  }
+
   const std::optional<std::string> ReplacementFunctionName =
       [&]() -> std::optional<std::string> {
     if (AnnexK) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
index 5adfee60d1a7d..63058c326ef29 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/UnsafeFunctionsCheck.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_UNSAFEFUNCTIONSCHECK_H
 
 #include "../ClangTidyCheck.h"
+#include "../utils/Matchers.h"
 #include <optional>
 
 namespace clang::tidy::bugprone {
@@ -32,7 +33,18 @@ class UnsafeFunctionsCheck : public ClangTidyCheck {
                            Preprocessor *ModuleExpanderPP) override;
   void onEndOfTranslationUnit() override;
 
+  struct CheckedFunction {
+    std::string Name;
+    matchers::MatchesAnyListedNameMatcher::NameMatcher Pattern;
+    std::string Replacement;
+    std::string Reason;
+  };
+
 private:
+  const std::vector<CheckedFunction> CustomFunctions;
+
+  // If true, the default set of functions are reported.
+  const bool ReportDefaultFunctions;
   /// If true, additional functions from widely used API-s (such as POSIX) are
   /// added to the list of reported functions.
   const bool ReportMoreUnsafeFunctions;
diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.h b/clang-tools-extra/clang-tidy/utils/Matchers.h
index 5fd98db967870..451c4ce92585b 100644
--- a/clang-tools-extra/clang-tidy/utils/Matchers.h
+++ b/clang-tools-extra/clang-tidy/utils/Matchers.h
@@ -85,15 +85,7 @@ class MatchesAnyListedNameMatcher
         NameList.begin(), NameList.end(), std::back_inserter(NameMatchers),
         [](const llvm::StringRef Name) { return NameMatcher(Name); });
   }
-  bool matches(
-      const NamedDecl &Node, ast_matchers::internal::ASTMatchFinder *Finder,
-      ast_matchers::internal::BoundNodesTreeBuilder *Builder) const override {
-    return llvm::any_of(NameMatchers, [&Node](const NameMatcher &NM) {
-      return NM.match(Node);
-    });
-  }
 
-private:
   class NameMatcher {
     llvm::Regex Regex;
     enum class MatchMode {
@@ -136,6 +128,15 @@ class MatchesAnyListedNameMatcher
     }
   };
 
+  bool matches(
+      const NamedDecl &Node, ast_matchers::internal::ASTMatchFinder *Finder,
+      ast_matchers::internal::BoundNodesTreeBuilder *Builder) const override {
+    return llvm::any_of(NameMatchers, [&Node](const NameMatcher &NM) {
+      return NM.match(Node);
+    });
+  }
+
+private:
   std::vector<NameMatcher> NameMatchers;
 };
 
diff --git a/clang-tools-extra/clangd/Diagnostics.cpp b/clang-tools-extra/clangd/Diagnostics.cpp
index a8214acc50558..a59d1e7ac8409 100644
--- a/clang-tools-extra/clangd/Diagnostics.cpp
+++ b/clang-tools-extra/clangd/Diagnostics.cpp
@@ -577,17 +577,7 @@ std::vector<Diag> StoreDiags::take(const clang::tidy::ClangTidyContext *Tidy) {
   for (auto &Diag : Output) {
     if (const char *ClangDiag = getDiagnosticCode(Diag.ID)) {
       // Warnings controlled by -Wfoo are better recognized by that name.
-      const StringRef Warning = [&] {
-        if (OrigSrcMgr) {
-          return OrigSrcMgr->getDiagnostics()
-              .getDiagnosticIDs()
-              ->getWarningOptionForDiag(Diag.ID);
-        }
-        if (!DiagnosticIDs::IsCustomDiag(Diag.ID))
-          return DiagnosticIDs{}.getWarningOptionForDiag(Diag.ID);
-        return StringRef{};
-      }();
-
+      StringRef Warning = DiagnosticIDs::getWarningOptionForDiag(Diag.ID);
       if (!Warning.empty()) {
         Diag.Name = ("-W" + Warning).str();
       } else {
@@ -904,23 +894,20 @@ void StoreDiags::flushLastDiag() {
   Output.push_back(std::move(*LastDiag));
 }
 
-bool isDiagnosticSuppressed(const clang::Diagnostic &Diag,
-                            const llvm::StringSet<> &Suppress,
-                            const LangOptions &LangOpts) {
+bool isBuiltinDiagnosticSuppressed(unsigned ID,
+                                   const llvm::StringSet<> &Suppress,
+                                   const LangOptions &LangOpts) {
   // Don't complain about header-only stuff in mainfiles if it's a header.
   // FIXME: would be cleaner to suppress in clang, once we decide whether the
   //        behavior should be to silently-ignore or respect the pragma.
-  if (Diag.getID() == diag::pp_pragma_sysheader_in_main_file &&
-      LangOpts.IsHeaderFile)
+  if (ID == diag::pp_pragma_sysheader_in_main_file && LangOpts.IsHeaderFile)
     return true;
 
-  if (const char *CodePtr = getDiagnosticCode(Diag.getID())) {
+  if (const char *CodePtr = getDiagnosticCode(ID)) {
     if (Suppress.contains(normalizeSuppressedCode(CodePtr)))
       return true;
   }
-  StringRef Warning =
-      Diag.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
-          Diag.getID());
+  StringRef Warning = DiagnosticIDs::getWarningOptionForDiag(ID);
   if (!Warning.empty() && Suppress.contains(Warning))
     return true;
   return false;
diff --git a/clang-tools-extra/clangd/Diagnostics.h b/clang-tools-extra/clangd/Diagnostics.h
index c45d8dc3aa6ce..d4c0478c63a5c 100644
--- a/clang-tools-extra/clangd/Diagnostics.h
+++ b/clang-tools-extra/clangd/Diagnostics.h
@@ -181,11 +181,11 @@ class StoreDiags : public DiagnosticConsumer {
 };
 
 /// Determine whether a (non-clang-tidy) diagnostic is suppressed by config.
-bool isDiagnosticSuppressed(const clang::Diagnostic &Diag,
-                            const llvm::StringSet<> &Suppressed,
-                            const LangOptions &);
+bool isBuiltinDiagnosticSuppressed(unsigned ID,
+                                   const llvm::StringSet<> &Suppressed,
+                                   const LangOptions &);
 /// Take a user-specified diagnostic code, and convert it to a normalized form
-/// stored in the config and consumed by isDiagnosticsSuppressed.
+/// stored in the config and consumed by isBuiltinDiagnosticsSuppressed.
 ///
 /// (This strips err_ and -W prefix so we can match with or without them.)
 llvm::StringRef normalizeSuppressedCode(llvm::StringRef);
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 5cf1691ce3961..045d32afbc938 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -342,7 +342,7 @@ void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs,
       if (Enable) {
         if (Diags.getDiagnosticLevel(ID, SourceLocation()) <
             DiagnosticsEngine::Warning) {
-          auto Group = Diags.getDiagnosticIDs()->getGroupForDiag(ID);
+          auto Group = DiagnosticIDs::getGroupForDiag(ID);
           if (!Group || !EnabledGroups(*Group))
             continue;
           Diags.setSeverity(ID, diag::Severity::Warning, SourceLocation());
@@ -585,8 +585,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
     ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
                                   const clang::Diagnostic &Info) {
       if (Cfg.Diagnostics.SuppressAll ||
-          isDiagnosticSuppressed(Info, Cfg.Diagnostics.Suppress,
-                                 Clang->getLangOpts()))
+          isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
+                                        Clang->getLangOpts()))
         return DiagnosticsEngine::Ignored;
 
       auto It = OverriddenSeverity.find(Info.getID());
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index 1fe534d78daec..c14c4d1ba103f 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -621,8 +621,8 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
                                            const clang::Diagnostic &Info) {
     if (Cfg.Diagnostics.SuppressAll ||
-        isDiagnosticSuppressed(Info, Cfg.Diagnostics.Suppress,
-                               CI.getLangOpts()))
+        isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
+                                      CI.getLangOpts()))
       return DiagnosticsEngine::Ignored;
     switch (Info.getID()) {
     case diag::warn_no_newline_eof:
diff --git a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
index cf9b42828568d..4ecfdf0184ab4 100644
--- a/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigCompileTests.cpp
@@ -298,41 +298,20 @@ TEST_F(ConfigCompileTests, DiagnosticSuppression) {
                                    "unreachable-code", "unused-variable",
                                    "typecheck_bool_condition",
                                    "unexpected_friend", "warn_alloca"));
-  clang::DiagnosticsEngine DiagEngine(new DiagnosticIDs, nullptr,
-                                      new clang::IgnoringDiagConsumer);
-
-  using Diag = clang::Diagnostic;
-  {
-    auto D = DiagEngine.Report(diag::warn_unreachable);
-    EXPECT_TRUE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
+      diag::warn_unreachable, Conf.Diagnostics.Suppress, LangOptions()));
   // Subcategory not respected/suppressed.
-  {
-    auto D = DiagEngine.Report(diag::warn_unreachable_break);
-    EXPECT_FALSE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
-  {
-    auto D = DiagEngine.Report(diag::warn_unused_variable);
-    EXPECT_TRUE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
-  {
-    auto D = DiagEngine.Report(diag::err_typecheck_bool_condition);
-    EXPECT_TRUE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
-  {
-    auto D = DiagEngine.Report(diag::err_unexpected_friend);
-    EXPECT_TRUE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
-  {
-    auto D = DiagEngine.Report(diag::warn_alloca);
-    EXPECT_TRUE(isDiagnosticSuppressed(
-        Diag{&DiagEngine, D}, Conf.Diagnostics.Suppress, LangOptions()));
-  }
+  EXPECT_FALSE(isBuiltinDiagnosticSuppressed(
+      diag::warn_unreachable_break, Conf.Diagnostics.Suppress, LangOptions()));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
+      diag::warn_unused_variable, Conf.Diagnostics.Suppress, LangOptions()));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(diag::err_typecheck_bool_condition,
+                                            Conf.Diagnostics.Suppress,
+                                            LangOptions()));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
+      diag::err_unexpected_friend, Conf.Diagnostics.Suppress, LangOptions()));
+  EXPECT_TRUE(isBuiltinDiagnosticSuppressed(
+      diag::warn_alloca, Conf.Diagnostics.Suppress, LangOptions()));
 
   Frag.Diagnostics.Suppress.emplace_back("*");
   EXPECT_TRUE(compileAndApply());
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index efb2e5ed2fbe1..7a47d6ebebf3b 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -1984,6 +1984,30 @@ TEST(Diagnostics, Tags) {
                 withTag(DiagnosticTag::Deprecated)))));
 }
 
+TEST(Diagnostics, TidyDiagsArentAffectedFromWerror) {
+  TestTU TU;
+  TU.ExtraArgs = {"-Werror"};
+  Annotations Test(R"cpp($typedef[[typedef int INT]]; // error-ok)cpp");
+  TU.Code = Test.code().str();
+  TU.ClangTidyProvider = addTidyChecks("modernize-use-using");
+  EXPECT_THAT(
+      TU.build().getDiagnostics(),
+      ifTidyChecks(UnorderedElementsAre(
+          AllOf(Diag(Test.range("typedef"), "use 'using' instead of 'typedef'"),
+                // Make sure severity for clang-tidy finding isn't bumped to
+                // error due to Werror in compile flags.
+                diagSeverity(DiagnosticsEngine::Warning)))));
+
+  TU.ClangTidyProvider =
+      addTidyChecks("modernize-use-using", /*WarningsAsErrors=*/"modernize-*");
+  EXPECT_THAT(
+      TU.build().getDiagnostics(),
+      ifTidyChecks(UnorderedElementsAre(
+          AllOf(Diag(Test.range("typedef"), "use 'using' instead of 'typedef'"),
+                // Unless bumped explicitly with WarnAsError.
+                diagSeverity(DiagnosticsEngine::Error)))));
+}
+
 TEST(Diagnostics, DeprecatedDiagsAreHints) {
   ClangdDiagnosticOptions Opts;
   std::optional<clangd::Diagnostic> Diag;
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 9a130a23b6e89..7d37a4b03222c 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -125,6 +125,10 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/forwarding-reference-overload>` check by fixing
   a crash when determining if an ``enable_if[_t]`` was found.
 
+- Improved :doc:`bugprone-posix-return
+  <clang-tidy/checks/bugprone/posix-return>` check to support integer literals
+  as LHS and posix call as RHS of comparison.
+
 - Improved :doc:`bugprone-sizeof-expression
   <clang-tidy/checks/bugprone/sizeof-expression>` check to find suspicious
   usages of ``sizeof()``, ``alignof()``, and ``offsetof()`` when adding or
@@ -135,6 +139,10 @@ Changes in existing checks
   `bsl::optional` and `bdlb::NullableValue` from
   <https://github.com/bloomberg/bde>_.
 
+- Improved :doc:`bugprone-unsafe-functions
+  <clang-tidy/checks/bugprone/unsafe-functions>` check to allow specifying
+  additional functions to match.
+
 - Improved :doc:`cert-flp30-c <clang-tidy/checks/cert/flp30-c>` check to
   fix false positive that floating point variable is only used in increment
   expression.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
index a0a267883b6fe..fb070627e31b1 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone/unsafe-functions.rst
@@ -19,6 +19,8 @@ The check implements the following rules from the CERT C Coding Standard:
 Unsafe functions
 ----------------
 
+The following functions are reported if :option:`ReportDefaultFunctions` is enabled.
+
 If *Annex K.* is available, a replacement from *Annex K.* is suggested for the
 following functions:
 
@@ -45,8 +47,7 @@ The following functions are always checked, regardless of *Annex K* availability
  - ``rewind``, suggested replacement: ``fseek``
  - ``setbuf``, suggested replacement: ``setvbuf``
 
-If `ReportMoreUnsafeFunctions
-<unsafe-functions.html#cmdoption-arg-ReportMoreUnsafeFunctions>`_ is enabled,
+If :option:`ReportMoreUnsafeFunctions` is enabled,
 the following functions are also checked:
 
  - ``bcmp``, suggested replacement: ``memcmp``
@@ -74,6 +75,44 @@ Both macros have to be defined to suggest replacement functions from *Annex K.*
 ``__STDC_WANT_LIB_EXT1__`` must be defined to ``1`` by the user **before**
 including any system headers.
 
+.. _CustomFunctions:
+
+Custom functions
+----------------
+
+The option :option:`CustomFunctions` allows the user to define custom functions to be
+checked. The format is the following, without newlines:
+
+.. code::
+
+   bugprone-unsafe-functions.CustomFunctions="
+     functionRegex1[, replacement1[, reason1]]; 
+     functionRegex2[, replacement2[, reason2]];
+     ...
+   "
+
+The functions are matched using POSIX extended regular expressions.
+*(Note: The regular expressions do not support negative* ``(?!)`` *matches.)*
+
+The `reason` is optional and is used to provide additional information about the
+reasoning behind the replacement. The default reason is `is marked as unsafe`.
+
+If `replacement` is empty, the text `it should not be used` will be shown
+instead of the suggestion for a replacement.
+
+As an example, the configuration `^original$, replacement, is deprecated;`
+will produce the following diagnostic message.
+
+.. code:: c
+  
+   original(); // warning: function 'original' is deprecated; 'replacement' should be used instead.
+   ::std::original(); // no-warning
+   original_function(); // no-warning
+
+If the regular expression contains the character `:`, it is matched against the
+qualified name (i.e. ``std::original``), otherwise the regex is matched against the unqualified name (``original``).
+If the regular expression starts with `::` (or `^::`), it is matched against the
+fully qualified name (``::std::original``).
 
 Options
 -------
@@ -86,6 +125,19 @@ Options
    this option enables.
    Default is `true`.
 
+.. option:: ReportDefaultFunctions
+
+    When `true`, the check reports the default set of functions.
+    Consider changing the setting to false if you only want to see custom
+    functions matched via :ref:`custom functions<CustomFunctions>`.
+    Default is `true`.
+
+.. option:: CustomFunctions
+
+    A semicolon-separated list of custom functions to be matched. A matched
+    function contains a regular expression, an optional name of the replacement
+    function, and an optional reason, separated by comma. For more information,
+    see :ref:`Custom functions<CustomFunctions>`.
 
 Examples
 --------
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/posix-return.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/posix-return.cpp
index 271893c707069..76d447a71d68b 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/posix-return.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/posix-return.cpp
@@ -74,6 +74,9 @@ void warningLessThanZero() {
   if (pthread_yield() < 0) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:23: warning:
   // CHECK-FIXES: pthread_yield() > 0
+  if (0 > pthread_yield() ) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning:
+  // CHECK-FIXES: 0 < pthread_yield()
 
 }
 
@@ -90,7 +93,8 @@ void warningAlwaysTrue() {
   // CHECK-MESSAGES: :[[@LINE-1]]:31: warning:
   if (pthread_yield() >= 0) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:23: warning:
-
+  if (0 <= pthread_yield()) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning:
 }
 
 void warningEqualsNegative() {
@@ -120,7 +124,14 @@ void warningEqualsNegative() {
   // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
   if (pthread_create(NULL, NULL, NULL, NULL) < -1) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
-
+  if (-1 == pthread_create(NULL, NULL, NULL, NULL)) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning:
+  if (-1 != pthread_create(NULL, NULL, NULL, NULL)) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning:
+  if (-1 >= pthread_create(NULL, NULL, NULL, NULL)) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning:
+  if (-1 > pthread_create(NULL, NULL, NULL, NULL)) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning:
 }
 
 void WarningWithMacro() {
@@ -162,6 +173,16 @@ void noWarning() {
   if (posix_openpt(0) < -1) {}
   if (posix_fadvise(0, 0, 0, 0) <= 0) {}
   if (posix_fadvise(0, 0, 0, 0) == 1) {}
+  if (0 > posix_openpt(0)) {}
+  if (0 >= posix_openpt(0)) {}
+  if (-1 == posix_openpt(0)) {}
+  if (-1 != posix_openpt(0)) {}
+  if (-1 >= posix_openpt(0)) {}
+  if (-1 > posix_openpt(0)) {}
+  if (posix_fadvise(0, 0, 0, 0) <= 0) {}
+  if (posix_fadvise(0, 0, 0, 0) == 1) {}
+  if (0 >= posix_fadvise(0, 0, 0, 0)) {}
+  if (1 == posix_fadvise(0, 0, 0, 0)) {}
 }
 
 namespace i {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp
new file mode 100644
index 0000000000000..fc97d1bc93bc5
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom-regex.cpp
@@ -0,0 +1,44 @@
+// RUN: %check_clang_tidy -check-suffix=NON-STRICT-REGEX         %s bugprone-unsafe-functions %t --\
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '::name_match,replacement,is a qualname match;^::prefix_match,,is matched on qualname prefix'}}"
+// RUN: %check_clang_tidy -check-suffix=STRICT-REGEX         %s bugprone-unsafe-functions %t --\
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '^name_match$,replacement,is matched on function name only;^::prefix_match$,,is a full qualname match'}}"
+
+void name_match();
+void prefix_match();
+
+namespace regex_test {
+void name_match();
+void prefix_match();
+}
+
+void name_match_regex();
+void prefix_match_regex();
+
+void f1() {
+  name_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match' is a qualname match; 'replacement' should be used instead
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'name_match' is matched on function name only; 'replacement' should be used instead
+  prefix_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match' is matched on qualname prefix; it should not be used
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'prefix_match' is a full qualname match; it should not be used
+
+  ::name_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match' is a qualname match; 'replacement' should be used instead
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'name_match' is matched on function name only; 'replacement' should be used instead
+  regex_test::name_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match' is a qualname match; 'replacement' should be used instead
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'name_match' is matched on function name only; 'replacement' should be used instead
+  name_match_regex();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match_regex' is a qualname match; 'replacement' should be used instead
+  // no-warning STRICT-REGEX
+
+  ::prefix_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match' is matched on qualname prefix; it should not be used
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'prefix_match' is a full qualname match; it should not be used
+  regex_test::prefix_match();
+  // no-warning NON-STRICT-REGEX
+  // no-warning STRICT-REGEX
+  prefix_match_regex();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match_regex' is matched on qualname prefix; it should not be used
+  // no-warning STRICT-REGEX
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom.c
new file mode 100644
index 0000000000000..7fd71ec2f2e7b
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions-custom.c
@@ -0,0 +1,27 @@
+// RUN: %check_clang_tidy -check-suffix=NON-STRICT-REGEX %s bugprone-unsafe-functions %t --\
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '::name_match,replacement,is a qualname match;^::prefix_match,,is matched on qualname prefix'}}"
+// RUN: %check_clang_tidy -check-suffix=STRICT-REGEX     %s bugprone-unsafe-functions %t --\
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.CustomFunctions: '^name_match$,replacement,is matched on function name only;^::prefix_match$,,is a full qualname match'}}"
+
+void name_match();
+void prefix_match();
+
+void name_match_regex();
+void prefix_match_regex();
+
+void f1() {
+  name_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match' is a qualname match; 'replacement' should be used instead
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'name_match' is matched on function name only; 'replacement' should be used instead
+  prefix_match();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match' is matched on qualname prefix; it should not be used
+  // CHECK-MESSAGES-STRICT-REGEX: :[[@LINE-2]]:3: warning: function 'prefix_match' is a full qualname match; it should not be used
+
+  name_match_regex();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'name_match_regex' is a qualname match; 'replacement' should be used instead
+  // no-warning STRICT-REGEX
+
+  prefix_match_regex();
+  // CHECK-MESSAGES-NON-STRICT-REGEX: :[[@LINE-1]]:3: warning: function 'prefix_match_regex' is matched on qualname prefix; it should not be used
+  // no-warning STRICT-REGEX
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
index 4bc2bad996d70..0409dd6bfcaa3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
@@ -12,6 +12,12 @@
 // RUN: %check_clang_tidy -check-suffix=WITH-ANNEX-K-CERT-ONLY  %s bugprone-unsafe-functions %t -- \
 // RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.ReportMoreUnsafeFunctions: false}}" \
 // RUN:                                                                                            -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
+// RUN: %check_clang_tidy -check-suffix=WITH-NONE-ENABLED       %s bugprone-unsafe-functions %t --\
+// RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.ReportDefaultFunctions: false}}" \
+// RUN:                                                                                            -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
+
+// CHECK-MESSAGES-WITH-NONE-ENABLED: 1 warning generated
+// CHECK-MESSAGES-WITH-NONE-ENABLED: Suppressed 1 warnings
 
 typedef __SIZE_TYPE__ size_t;
 typedef __WCHAR_TYPE__ wchar_t;
diff --git a/clang/CodeOwners.rst b/clang/CodeOwners.rst
index 2ae04c129eb76..f067b7183ae73 100644
--- a/clang/CodeOwners.rst
+++ b/clang/CodeOwners.rst
@@ -120,7 +120,7 @@ OpenBSD driver
 Driver parts not covered by someone else
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 | Fangrui Song
-| maskray\@google.com (email), MaskRay (Phabricator), MaskRay (GitHub)
+| i\@maskray.me (email), MaskRay (Phabricator), MaskRay (GitHub)
 
 
 Tools
diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 4da99e899e7f7..f8a20a1e22472 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -133,7 +133,7 @@ def from_param(cls, param: str | bytes | None) -> c_interop_string:
         )
 
     @staticmethod
-    def to_python_string(x: c_interop_string, *args: Any) -> str | None:
+    def to_python_string(x: c_interop_string) -> str | None:
         return x.value
 
 
@@ -241,9 +241,9 @@ def __del__(self) -> None:
         conf.lib.clang_disposeString(self)
 
     @staticmethod
-    def from_result(res: _CXString, fn: Any = None, args: Any = None) -> str:
+    def from_result(res: _CXString) -> str:
         assert isinstance(res, _CXString)
-        pystr: str | None = conf.lib.clang_getCString(res)
+        pystr = c_interop_string.to_python_string(conf.lib.clang_getCString(res))
         if pystr is None:
             return ""
         return pystr
@@ -424,7 +424,7 @@ def location(self):
 
     @property
     def spelling(self):
-        return conf.lib.clang_getDiagnosticSpelling(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getDiagnosticSpelling(self))
 
     @property
     def ranges(self) -> NoSliceSequence[SourceRange]:
@@ -453,7 +453,9 @@ def __len__(self) -> int:
 
             def __getitem__(self, key: int) -> FixIt:
                 range = SourceRange()
-                value = conf.lib.clang_getDiagnosticFixIt(self.diag, key, byref(range))
+                value = _CXString.from_result(
+                    conf.lib.clang_getDiagnosticFixIt(self.diag, key, byref(range))
+                )
                 if len(value) == 0:
                     raise IndexError
 
@@ -486,12 +488,12 @@ def category_number(self):
     @property
     def category_name(self):
         """The string name of the category for this diagnostic."""
-        return conf.lib.clang_getDiagnosticCategoryText(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getDiagnosticCategoryText(self))
 
     @property
     def option(self):
         """The command-line option that enables this diagnostic."""
-        return conf.lib.clang_getDiagnosticOption(self, None)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getDiagnosticOption(self, None))
 
     @property
     def disable_option(self):
@@ -511,7 +513,7 @@ def format(self, options=None):
             options = conf.lib.clang_defaultDiagnosticDisplayOptions()
         if options & ~Diagnostic._FormatOptionsMask:
             raise ValueError("Invalid format options")
-        return conf.lib.clang_formatDiagnostic(self, options)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_formatDiagnostic(self, options))
 
     def __repr__(self):
         return "<Diagnostic severity %r, location %r, spelling %r>" % (
@@ -1734,7 +1736,7 @@ def get_definition(self):
         """
         # TODO: Should probably check that this is either a reference or
         # declaration prior to issuing the lookup.
-        return conf.lib.clang_getCursorDefinition(self)  # type: ignore [no-any-return]
+        return Cursor.from_result(conf.lib.clang_getCursorDefinition(self), self)
 
     def get_usr(self):
         """Return the Unified Symbol Resolution (USR) for the entity referenced
@@ -1745,13 +1747,13 @@ def get_usr(self):
         program. USRs can be compared across translation units to determine,
         e.g., when references in one translation refer to an entity defined in
         another translation unit."""
-        return conf.lib.clang_getCursorUSR(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getCursorUSR(self))
 
     def get_included_file(self):
         """Returns the File that is included by the current inclusion cursor."""
         assert self.kind == CursorKind.INCLUSION_DIRECTIVE
 
-        return conf.lib.clang_getIncludedFile(self)  # type: ignore [no-any-return]
+        return File.from_result(conf.lib.clang_getIncludedFile(self), self)
 
     @property
     def kind(self):
@@ -1762,7 +1764,9 @@ def kind(self):
     def spelling(self):
         """Return the spelling of the entity pointed at by the cursor."""
         if not hasattr(self, "_spelling"):
-            self._spelling = conf.lib.clang_getCursorSpelling(self)
+            self._spelling = _CXString.from_result(
+                conf.lib.clang_getCursorSpelling(self)
+            )
 
         return self._spelling
 
@@ -1776,7 +1780,9 @@ def displayname(self):
         arguments of a class template specialization.
         """
         if not hasattr(self, "_displayname"):
-            self._displayname = conf.lib.clang_getCursorDisplayName(self)
+            self._displayname = _CXString.from_result(
+                conf.lib.clang_getCursorDisplayName(self)
+            )
 
         return self._displayname
 
@@ -1784,7 +1790,9 @@ def displayname(self):
     def mangled_name(self):
         """Return the mangled name for the entity referenced by this cursor."""
         if not hasattr(self, "_mangled_name"):
-            self._mangled_name = conf.lib.clang_Cursor_getMangling(self)
+            self._mangled_name = _CXString.from_result(
+                conf.lib.clang_Cursor_getMangling(self)
+            )
 
         return self._mangled_name
 
@@ -1876,7 +1884,7 @@ def type(self):
         Retrieve the Type (if any) of the entity pointed at by the cursor.
         """
         if not hasattr(self, "_type"):
-            self._type = conf.lib.clang_getCursorType(self)
+            self._type = Type.from_result(conf.lib.clang_getCursorType(self), (self,))
 
         return self._type
 
@@ -1890,7 +1898,9 @@ def canonical(self):
         declarations will be identical.
         """
         if not hasattr(self, "_canonical"):
-            self._canonical = conf.lib.clang_getCanonicalCursor(self)
+            self._canonical = Cursor.from_cursor_result(
+                conf.lib.clang_getCanonicalCursor(self), self
+            )
 
         return self._canonical
 
@@ -1898,7 +1908,9 @@ def canonical(self):
     def result_type(self):
         """Retrieve the Type of the result for this Cursor."""
         if not hasattr(self, "_result_type"):
-            self._result_type = conf.lib.clang_getCursorResultType(self)
+            self._result_type = Type.from_result(
+                conf.lib.clang_getCursorResultType(self), (self,)
+            )
 
         return self._result_type
 
@@ -1925,7 +1937,9 @@ def underlying_typedef_type(self):
         """
         if not hasattr(self, "_underlying_type"):
             assert self.kind.is_declaration()
-            self._underlying_type = conf.lib.clang_getTypedefDeclUnderlyingType(self)
+            self._underlying_type = Type.from_result(
+                conf.lib.clang_getTypedefDeclUnderlyingType(self), (self,)
+            )
 
         return self._underlying_type
 
@@ -1938,7 +1952,9 @@ def enum_type(self):
         """
         if not hasattr(self, "_enum_type"):
             assert self.kind == CursorKind.ENUM_DECL
-            self._enum_type = conf.lib.clang_getEnumDeclIntegerType(self)
+            self._enum_type = Type.from_result(
+                conf.lib.clang_getEnumDeclIntegerType(self), (self,)
+            )
 
         return self._enum_type
 
@@ -1972,7 +1988,9 @@ def enum_value(self):
     def objc_type_encoding(self):
         """Return the Objective-C type encoding as a str."""
         if not hasattr(self, "_objc_type_encoding"):
-            self._objc_type_encoding = conf.lib.clang_getDeclObjCTypeEncoding(self)
+            self._objc_type_encoding = _CXString.from_result(
+                conf.lib.clang_getDeclObjCTypeEncoding(self)
+            )
 
         return self._objc_type_encoding
 
@@ -1988,7 +2006,9 @@ def hash(self):
     def semantic_parent(self):
         """Return the semantic parent for this cursor."""
         if not hasattr(self, "_semantic_parent"):
-            self._semantic_parent = conf.lib.clang_getCursorSemanticParent(self)
+            self._semantic_parent = Cursor.from_cursor_result(
+                conf.lib.clang_getCursorSemanticParent(self), self
+            )
 
         return self._semantic_parent
 
@@ -1996,7 +2016,9 @@ def semantic_parent(self):
     def lexical_parent(self):
         """Return the lexical parent for this cursor."""
         if not hasattr(self, "_lexical_parent"):
-            self._lexical_parent = conf.lib.clang_getCursorLexicalParent(self)
+            self._lexical_parent = Cursor.from_cursor_result(
+                conf.lib.clang_getCursorLexicalParent(self), self
+            )
 
         return self._lexical_parent
 
@@ -2014,25 +2036,27 @@ def referenced(self):
         representing the entity that it references.
         """
         if not hasattr(self, "_referenced"):
-            self._referenced = conf.lib.clang_getCursorReferenced(self)
+            self._referenced = Cursor.from_result(
+                conf.lib.clang_getCursorReferenced(self), self
+            )
 
         return self._referenced
 
     @property
     def brief_comment(self):
         """Returns the brief comment text associated with that Cursor"""
-        return conf.lib.clang_Cursor_getBriefCommentText(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_Cursor_getBriefCommentText(self))
 
     @property
     def raw_comment(self):
         """Returns the raw comment text associated with that Cursor"""
-        return conf.lib.clang_Cursor_getRawCommentText(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_Cursor_getRawCommentText(self))
 
     def get_arguments(self):
         """Return an iterator for accessing the arguments of this cursor."""
         num_args = conf.lib.clang_Cursor_getNumArguments(self)
         for i in range(0, num_args):
-            yield conf.lib.clang_Cursor_getArgument(self, i)
+            yield Cursor.from_result(conf.lib.clang_Cursor_getArgument(self, i), self)
 
     def get_num_template_arguments(self):
         """Returns the number of template args associated with this cursor."""
@@ -2041,11 +2065,15 @@ def get_num_template_arguments(self):
     def get_template_argument_kind(self, num):
         """Returns the TemplateArgumentKind for the indicated template
         argument."""
-        return conf.lib.clang_Cursor_getTemplateArgumentKind(self, num)  # type: ignore [no-any-return]
+        return TemplateArgumentKind.from_id(
+            conf.lib.clang_Cursor_getTemplateArgumentKind(self, num)
+        )
 
     def get_template_argument_type(self, num):
         """Returns the CXType for the indicated template argument."""
-        return conf.lib.clang_Cursor_getTemplateArgumentType(self, num)  # type: ignore [no-any-return]
+        return Type.from_result(
+            conf.lib.clang_Cursor_getTemplateArgumentType(self, num), (self, num)
+        )
 
     def get_template_argument_value(self, num):
         """Returns the value of the indicated arg as a signed 64b integer."""
@@ -2116,7 +2144,7 @@ def get_bitfield_width(self):
         return conf.lib.clang_getFieldDeclBitWidth(self)  # type: ignore [no-any-return]
 
     @staticmethod
-    def from_result(res, fn, args):
+    def from_result(res, arg):
         assert isinstance(res, Cursor)
         # FIXME: There should just be an isNull method.
         if res == conf.lib.clang_getNullCursor():
@@ -2125,14 +2153,10 @@ def from_result(res, fn, args):
         # Store a reference to the TU in the Python object so it won't get GC'd
         # before the Cursor.
         tu = None
-        for arg in args:
-            if isinstance(arg, TranslationUnit):
-                tu = arg
-                break
-
-            if hasattr(arg, "translation_unit"):
-                tu = arg.translation_unit
-                break
+        if isinstance(arg, TranslationUnit):
+            tu = arg
+        elif hasattr(arg, "translation_unit"):
+            tu = arg.translation_unit
 
         assert tu is not None
 
@@ -2140,12 +2164,12 @@ def from_result(res, fn, args):
         return res
 
     @staticmethod
-    def from_cursor_result(res, fn, args):
+    def from_cursor_result(res, arg):
         assert isinstance(res, Cursor)
         if res == conf.lib.clang_getNullCursor():
             return None
 
-        res._tu = args[0]._tu
+        res._tu = arg._tu
         return res
 
 
@@ -2250,7 +2274,7 @@ class TypeKind(BaseEnumeration):
     @property
     def spelling(self):
         """Retrieve the spelling of this TypeKind."""
-        return conf.lib.clang_getTypeKindSpelling(self.value)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getTypeKindSpelling(self.value))
 
     INVALID = 0
     UNEXPOSED = 1
@@ -2438,7 +2462,9 @@ def __getitem__(self, key: int) -> Type:
                         "%d > %d" % (key, len(self))
                     )
 
-                result: Type = conf.lib.clang_getArgType(self.parent, key)
+                result = Type.from_result(
+                    conf.lib.clang_getArgType(self.parent, key), (self.parent, key)
+                )
                 if result.kind == TypeKind.INVALID:
                     raise IndexError("Argument could not be retrieved.")
 
@@ -2454,7 +2480,7 @@ def element_type(self):
         If accessed on a type that is not an array, complex, or vector type, an
         exception will be raised.
         """
-        result = conf.lib.clang_getElementType(self)
+        result = Type.from_result(conf.lib.clang_getElementType(self), (self,))
         if result.kind == TypeKind.INVALID:
             raise Exception("Element type not available on this type.")
 
@@ -2482,7 +2508,7 @@ def translation_unit(self):
         return self._tu
 
     @staticmethod
-    def from_result(res, fn, args):
+    def from_result(res, args):
         assert isinstance(res, Type)
 
         tu = None
@@ -2500,7 +2526,9 @@ def get_num_template_arguments(self):
         return conf.lib.clang_Type_getNumTemplateArguments(self)  # type: ignore [no-any-return]
 
     def get_template_argument_type(self, num):
-        return conf.lib.clang_Type_getTemplateArgumentAsType(self, num)  # type: ignore [no-any-return]
+        return Type.from_result(
+            conf.lib.clang_Type_getTemplateArgumentAsType(self, num), (self, num)
+        )
 
     def get_canonical(self):
         """
@@ -2512,7 +2540,7 @@ def get_canonical(self):
         example, if 'T' is a typedef for 'int', the canonical type for
         'T' would be 'int'.
         """
-        return conf.lib.clang_getCanonicalType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_getCanonicalType(self), (self,))
 
     def is_const_qualified(self):
         """Determine whether a Type has the "const" qualifier set.
@@ -2548,7 +2576,7 @@ def get_address_space(self):
         return conf.lib.clang_getAddressSpace(self)  # type: ignore [no-any-return]
 
     def get_typedef_name(self):
-        return conf.lib.clang_getTypedefName(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getTypedefName(self))
 
     def is_pod(self):
         """Determine whether this Type represents plain old data (POD)."""
@@ -2558,25 +2586,25 @@ def get_pointee(self):
         """
         For pointer types, returns the type of the pointee.
         """
-        return conf.lib.clang_getPointeeType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_getPointeeType(self), (self,))
 
     def get_declaration(self):
         """
         Return the cursor for the declaration of the given type.
         """
-        return conf.lib.clang_getTypeDeclaration(self)  # type: ignore [no-any-return]
+        return Cursor.from_result(conf.lib.clang_getTypeDeclaration(self), self)
 
     def get_result(self):
         """
         Retrieve the result type associated with a function type.
         """
-        return conf.lib.clang_getResultType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_getResultType(self), (self,))
 
     def get_array_element_type(self):
         """
         Retrieve the type of the elements of the array type.
         """
-        return conf.lib.clang_getArrayElementType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_getArrayElementType(self), (self,))
 
     def get_array_size(self):
         """
@@ -2588,13 +2616,13 @@ def get_class_type(self):
         """
         Retrieve the class type of the member pointer type.
         """
-        return conf.lib.clang_Type_getClassType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_Type_getClassType(self), (self,))
 
     def get_named_type(self):
         """
         Retrieve the type named by the qualified-id.
         """
-        return conf.lib.clang_Type_getNamedType(self)  # type: ignore [no-any-return]
+        return Type.from_result(conf.lib.clang_Type_getNamedType(self), (self,))
 
     def get_align(self):
         """
@@ -2647,7 +2675,7 @@ def get_exception_specification_kind(self):
     @property
     def spelling(self):
         """Retrieve the spelling of this Type."""
-        return conf.lib.clang_getTypeSpelling(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getTypeSpelling(self))
 
     def __eq__(self, other):
         if type(other) != type(self):
@@ -2737,7 +2765,9 @@ def __repr__(self):
     def spelling(self):
         if self.__kindNumber in SpellingCache:
             return SpellingCache[self.__kindNumber]
-        return conf.lib.clang_getCompletionChunkText(self.cs, self.key)  # type: ignore [no-any-return]
+        return _CXString.from_result(
+            conf.lib.clang_getCompletionChunkText(self.cs, self.key)
+        )
 
     # We do not use @CachedProperty here, as the manual implementation is
     # apparently still significantly faster. Please profile carefully if you
@@ -2839,7 +2869,9 @@ def availability(self):
     @property
     def briefComment(self):
         if conf.function_exists("clang_getCompletionBriefComment"):
-            return conf.lib.clang_getCompletionBriefComment(self.obj)  # type: ignore [no-any-return]
+            return _CXString.from_result(
+                conf.lib.clang_getCompletionBriefComment(self.obj)
+            )
         return _CXString()
 
     def __repr__(self):
@@ -3125,12 +3157,12 @@ def __del__(self):
     @property
     def cursor(self):
         """Retrieve the cursor that represents the given translation unit."""
-        return conf.lib.clang_getTranslationUnitCursor(self)  # type: ignore [no-any-return]
+        return Cursor.from_result(conf.lib.clang_getTranslationUnitCursor(self), self)
 
     @property
     def spelling(self):
         """Get the original translation unit source file name."""
-        return conf.lib.clang_getTranslationUnitSpelling(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getTranslationUnitSpelling(self))
 
     def get_includes(self):
         """
@@ -3356,7 +3388,7 @@ def from_name(translation_unit, file_name):
     @property
     def name(self):
         """Return the complete file and path name of the file."""
-        return conf.lib.clang_getFileName(self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getFileName(self))
 
     @property
     def time(self):
@@ -3370,12 +3402,12 @@ def __repr__(self):
         return "<File: %s>" % (self.name)
 
     @staticmethod
-    def from_result(res, fn, args):
+    def from_result(res, arg):
         assert isinstance(res, c_object_p)
         res = File(res)
 
         # Copy a reference to the TranslationUnit to prevent premature GC.
-        res._tu = args[0]._tu
+        res._tu = arg._tu
         return res
 
 
@@ -3440,12 +3472,16 @@ def __init__(self, cmd, ccmds):
     @property
     def directory(self):
         """Get the working directory for this CompileCommand"""
-        return conf.lib.clang_CompileCommand_getDirectory(self.cmd)  # type: ignore [no-any-return]
+        return _CXString.from_result(
+            conf.lib.clang_CompileCommand_getDirectory(self.cmd)
+        )
 
     @property
     def filename(self):
         """Get the working filename for this CompileCommand"""
-        return conf.lib.clang_CompileCommand_getFilename(self.cmd)  # type: ignore [no-any-return]
+        return _CXString.from_result(
+            conf.lib.clang_CompileCommand_getFilename(self.cmd)
+        )
 
     @property
     def arguments(self):
@@ -3457,7 +3493,9 @@ def arguments(self):
         """
         length = conf.lib.clang_CompileCommand_getNumArgs(self.cmd)
         for i in range(length):
-            yield conf.lib.clang_CompileCommand_getArg(self.cmd, i)
+            yield _CXString.from_result(
+                conf.lib.clang_CompileCommand_getArg(self.cmd, i)
+            )
 
 
 class CompileCommands:
@@ -3482,7 +3520,7 @@ def __getitem__(self, i):
         return CompileCommand(cc, self)
 
     @staticmethod
-    def from_result(res, fn, args):
+    def from_result(res):
         if not res:
             return None
         return CompileCommands(res)
@@ -3500,7 +3538,7 @@ def __del__(self):
         conf.lib.clang_CompilationDatabase_dispose(self)
 
     @staticmethod
-    def from_result(res, fn, args):
+    def from_result(res):
         if not res:
             raise CompilationDatabaseError(0, "CompilationDatabase loading failed")
         return CompilationDatabase(res)
@@ -3510,8 +3548,10 @@ def fromDirectory(buildDir):
         """Builds a CompilationDatabase from the database found in buildDir"""
         errorCode = c_uint()
         try:
-            cdb = conf.lib.clang_CompilationDatabase_fromDirectory(
-                os.fspath(buildDir), byref(errorCode)
+            cdb = CompilationDatabase.from_result(
+                conf.lib.clang_CompilationDatabase_fromDirectory(
+                    os.fspath(buildDir), byref(errorCode)
+                )
             )
         except CompilationDatabaseError as e:
             raise CompilationDatabaseError(
@@ -3524,8 +3564,10 @@ def getCompileCommands(self, filename):
         Get an iterable object providing all the CompileCommands available to
         build filename. Returns None if filename is not found in the database.
         """
-        return conf.lib.clang_CompilationDatabase_getCompileCommands(  # type: ignore [no-any-return]
-            self, os.fspath(filename)
+        return CompileCommands.from_result(
+            conf.lib.clang_CompilationDatabase_getCompileCommands(  # type: ignore [no-any-return]
+                self, os.fspath(filename)
+            )
         )
 
     def getAllCompileCommands(self):
@@ -3533,7 +3575,9 @@ def getAllCompileCommands(self):
         Get an iterable object providing all the CompileCommands available from
         the database.
         """
-        return conf.lib.clang_CompilationDatabase_getAllCompileCommands(self)  # type: ignore [no-any-return]
+        return CompileCommands.from_result(
+            conf.lib.clang_CompilationDatabase_getAllCompileCommands(self)  # type: ignore [no-any-return]
+        )
 
 
 class Token(Structure):
@@ -3554,7 +3598,7 @@ def spelling(self):
 
         This is the textual representation of the token in source.
         """
-        return conf.lib.clang_getTokenSpelling(self._tu, self)  # type: ignore [no-any-return]
+        return _CXString.from_result(conf.lib.clang_getTokenSpelling(self._tu, self))
 
     @property
     def kind(self):
@@ -3661,41 +3705,19 @@ def write_main_file_to_stdout(self):
         "clang_CompilationDatabase_fromDirectory",
         [c_interop_string, POINTER(c_uint)],
         c_object_p,
-        CompilationDatabase.from_result,
-    ),
-    (
-        "clang_CompilationDatabase_getAllCompileCommands",
-        [c_object_p],
-        c_object_p,
-        CompileCommands.from_result,
     ),
+    ("clang_CompilationDatabase_getAllCompileCommands", [c_object_p], c_object_p),
     (
         "clang_CompilationDatabase_getCompileCommands",
         [c_object_p, c_interop_string],
         c_object_p,
-        CompileCommands.from_result,
     ),
     ("clang_CompileCommands_dispose", [c_object_p]),
     ("clang_CompileCommands_getCommand", [c_object_p, c_uint], c_object_p),
     ("clang_CompileCommands_getSize", [c_object_p], c_uint),
-    (
-        "clang_CompileCommand_getArg",
-        [c_object_p, c_uint],
-        _CXString,
-        _CXString.from_result,
-    ),
-    (
-        "clang_CompileCommand_getDirectory",
-        [c_object_p],
-        _CXString,
-        _CXString.from_result,
-    ),
-    (
-        "clang_CompileCommand_getFilename",
-        [c_object_p],
-        _CXString,
-        _CXString.from_result,
-    ),
+    ("clang_CompileCommand_getArg", [c_object_p, c_uint], _CXString),
+    ("clang_CompileCommand_getDirectory", [c_object_p], _CXString),
+    ("clang_CompileCommand_getFilename", [c_object_p], _CXString),
     ("clang_CompileCommand_getNumArgs", [c_object_p], c_uint),
     (
         "clang_codeCompleteAt",
@@ -3743,82 +3765,62 @@ def write_main_file_to_stdout(self):
     ("clang_equalLocations", [SourceLocation, SourceLocation], bool),
     ("clang_equalRanges", [SourceRange, SourceRange], bool),
     ("clang_equalTypes", [Type, Type], bool),
-    ("clang_formatDiagnostic", [Diagnostic, c_uint], _CXString, _CXString.from_result),
-    ("clang_getArgType", [Type, c_uint], Type, Type.from_result),
-    ("clang_getArrayElementType", [Type], Type, Type.from_result),
+    ("clang_formatDiagnostic", [Diagnostic, c_uint], _CXString),
+    ("clang_getArgType", [Type, c_uint], Type),
+    ("clang_getArrayElementType", [Type], Type),
     ("clang_getArraySize", [Type], c_longlong),
     ("clang_getFieldDeclBitWidth", [Cursor], c_int),
-    ("clang_getCanonicalCursor", [Cursor], Cursor, Cursor.from_cursor_result),
-    ("clang_getCanonicalType", [Type], Type, Type.from_result),
+    ("clang_getCanonicalCursor", [Cursor], Cursor),
+    ("clang_getCanonicalType", [Type], Type),
     ("clang_getChildDiagnostics", [Diagnostic], c_object_p),
     ("clang_getCompletionAvailability", [c_void_p], c_int),
-    ("clang_getCompletionBriefComment", [c_void_p], _CXString, _CXString.from_result),
+    ("clang_getCompletionBriefComment", [c_void_p], _CXString),
     ("clang_getCompletionChunkCompletionString", [c_void_p, c_int], c_object_p),
     ("clang_getCompletionChunkKind", [c_void_p, c_int], c_int),
-    (
-        "clang_getCompletionChunkText",
-        [c_void_p, c_int],
-        _CXString,
-        _CXString.from_result,
-    ),
+    ("clang_getCompletionChunkText", [c_void_p, c_int], _CXString),
     ("clang_getCompletionPriority", [c_void_p], c_int),
-    (
-        "clang_getCString",
-        [_CXString],
-        c_interop_string,
-        c_interop_string.to_python_string,
-    ),
+    ("clang_getCString", [_CXString], c_interop_string),
     ("clang_getCursor", [TranslationUnit, SourceLocation], Cursor),
     ("clang_getCursorAvailability", [Cursor], c_int),
-    ("clang_getCursorDefinition", [Cursor], Cursor, Cursor.from_result),
-    ("clang_getCursorDisplayName", [Cursor], _CXString, _CXString.from_result),
+    ("clang_getCursorDefinition", [Cursor], Cursor),
+    ("clang_getCursorDisplayName", [Cursor], _CXString),
     ("clang_getCursorExtent", [Cursor], SourceRange),
-    ("clang_getCursorLexicalParent", [Cursor], Cursor, Cursor.from_cursor_result),
+    ("clang_getCursorLexicalParent", [Cursor], Cursor),
     ("clang_getCursorLocation", [Cursor], SourceLocation),
-    ("clang_getCursorReferenced", [Cursor], Cursor, Cursor.from_result),
+    ("clang_getCursorReferenced", [Cursor], Cursor),
     ("clang_getCursorReferenceNameRange", [Cursor, c_uint, c_uint], SourceRange),
-    ("clang_getCursorResultType", [Cursor], Type, Type.from_result),
-    ("clang_getCursorSemanticParent", [Cursor], Cursor, Cursor.from_cursor_result),
-    ("clang_getCursorSpelling", [Cursor], _CXString, _CXString.from_result),
-    ("clang_getCursorType", [Cursor], Type, Type.from_result),
-    ("clang_getCursorUSR", [Cursor], _CXString, _CXString.from_result),
-    ("clang_Cursor_getMangling", [Cursor], _CXString, _CXString.from_result),
+    ("clang_getCursorResultType", [Cursor], Type),
+    ("clang_getCursorSemanticParent", [Cursor], Cursor),
+    ("clang_getCursorSpelling", [Cursor], _CXString),
+    ("clang_getCursorType", [Cursor], Type),
+    ("clang_getCursorUSR", [Cursor], _CXString),
+    ("clang_Cursor_getMangling", [Cursor], _CXString),
     # ("clang_getCXTUResourceUsage",
     #  [TranslationUnit],
     #  CXTUResourceUsage),
     ("clang_getCXXAccessSpecifier", [Cursor], c_uint),
-    ("clang_getDeclObjCTypeEncoding", [Cursor], _CXString, _CXString.from_result),
+    ("clang_getDeclObjCTypeEncoding", [Cursor], _CXString),
     ("clang_getDiagnostic", [c_object_p, c_uint], c_object_p),
     ("clang_getDiagnosticCategory", [Diagnostic], c_uint),
-    ("clang_getDiagnosticCategoryText", [Diagnostic], _CXString, _CXString.from_result),
-    (
-        "clang_getDiagnosticFixIt",
-        [Diagnostic, c_uint, POINTER(SourceRange)],
-        _CXString,
-        _CXString.from_result,
-    ),
+    ("clang_getDiagnosticCategoryText", [Diagnostic], _CXString),
+    ("clang_getDiagnosticFixIt", [Diagnostic, c_uint, POINTER(SourceRange)], _CXString),
     ("clang_getDiagnosticInSet", [c_object_p, c_uint], c_object_p),
     ("clang_getDiagnosticLocation", [Diagnostic], SourceLocation),
     ("clang_getDiagnosticNumFixIts", [Diagnostic], c_uint),
     ("clang_getDiagnosticNumRanges", [Diagnostic], c_uint),
-    (
-        "clang_getDiagnosticOption",
-        [Diagnostic, POINTER(_CXString)],
-        _CXString,
-        _CXString.from_result,
-    ),
+    ("clang_getDiagnosticOption", [Diagnostic, POINTER(_CXString)], _CXString),
     ("clang_getDiagnosticRange", [Diagnostic, c_uint], SourceRange),
     ("clang_getDiagnosticSeverity", [Diagnostic], c_int),
-    ("clang_getDiagnosticSpelling", [Diagnostic], _CXString, _CXString.from_result),
-    ("clang_getElementType", [Type], Type, Type.from_result),
+    ("clang_getDiagnosticSpelling", [Diagnostic], _CXString),
+    ("clang_getElementType", [Type], Type),
     ("clang_getEnumConstantDeclUnsignedValue", [Cursor], c_ulonglong),
     ("clang_getEnumConstantDeclValue", [Cursor], c_longlong),
-    ("clang_getEnumDeclIntegerType", [Cursor], Type, Type.from_result),
+    ("clang_getEnumDeclIntegerType", [Cursor], Type),
     ("clang_getFile", [TranslationUnit, c_interop_string], c_object_p),
-    ("clang_getFileName", [File], _CXString, _CXString.from_result),
+    ("clang_getFileName", [File], _CXString),
     ("clang_getFileTime", [File], c_uint),
-    ("clang_getIBOutletCollectionType", [Cursor], Type, Type.from_result),
-    ("clang_getIncludedFile", [Cursor], c_object_p, File.from_result),
+    ("clang_getIBOutletCollectionType", [Cursor], Type),
+    ("clang_getIncludedFile", [Cursor], c_object_p),
     (
         "clang_getInclusions",
         [TranslationUnit, translation_unit_includes_callback, py_object],
@@ -3842,41 +3844,26 @@ def write_main_file_to_stdout(self):
     ("clang_getNumDiagnosticsInSet", [c_object_p], c_uint),
     ("clang_getNumElements", [Type], c_longlong),
     ("clang_getNumOverloadedDecls", [Cursor], c_uint),
-    ("clang_getOverloadedDecl", [Cursor, c_uint], Cursor, Cursor.from_cursor_result),
-    ("clang_getPointeeType", [Type], Type, Type.from_result),
+    ("clang_getOverloadedDecl", [Cursor, c_uint], Cursor),
+    ("clang_getPointeeType", [Type], Type),
     ("clang_getRange", [SourceLocation, SourceLocation], SourceRange),
     ("clang_getRangeEnd", [SourceRange], SourceLocation),
     ("clang_getRangeStart", [SourceRange], SourceLocation),
-    ("clang_getResultType", [Type], Type, Type.from_result),
-    ("clang_getSpecializedCursorTemplate", [Cursor], Cursor, Cursor.from_cursor_result),
+    ("clang_getResultType", [Type], Type),
+    ("clang_getSpecializedCursorTemplate", [Cursor], Cursor),
     ("clang_getTemplateCursorKind", [Cursor], c_uint),
     ("clang_getTokenExtent", [TranslationUnit, Token], SourceRange),
     ("clang_getTokenKind", [Token], c_uint),
     ("clang_getTokenLocation", [TranslationUnit, Token], SourceLocation),
-    (
-        "clang_getTokenSpelling",
-        [TranslationUnit, Token],
-        _CXString,
-        _CXString.from_result,
-    ),
-    ("clang_getTranslationUnitCursor", [TranslationUnit], Cursor, Cursor.from_result),
-    (
-        "clang_getTranslationUnitSpelling",
-        [TranslationUnit],
-        _CXString,
-        _CXString.from_result,
-    ),
-    (
-        "clang_getTUResourceUsageName",
-        [c_uint],
-        c_interop_string,
-        c_interop_string.to_python_string,
-    ),
-    ("clang_getTypeDeclaration", [Type], Cursor, Cursor.from_result),
-    ("clang_getTypedefDeclUnderlyingType", [Cursor], Type, Type.from_result),
-    ("clang_getTypedefName", [Type], _CXString, _CXString.from_result),
-    ("clang_getTypeKindSpelling", [c_uint], _CXString, _CXString.from_result),
-    ("clang_getTypeSpelling", [Type], _CXString, _CXString.from_result),
+    ("clang_getTokenSpelling", [TranslationUnit, Token], _CXString),
+    ("clang_getTranslationUnitCursor", [TranslationUnit], Cursor),
+    ("clang_getTranslationUnitSpelling", [TranslationUnit], _CXString),
+    ("clang_getTUResourceUsageName", [c_uint], c_interop_string),
+    ("clang_getTypeDeclaration", [Type], Cursor),
+    ("clang_getTypedefDeclUnderlyingType", [Cursor], Type),
+    ("clang_getTypedefName", [Type], _CXString),
+    ("clang_getTypeKindSpelling", [c_uint], _CXString),
+    ("clang_getTypeSpelling", [Type], _CXString),
     ("clang_hashCursor", [Cursor], c_uint),
     ("clang_isAttribute", [CursorKind], bool),
     ("clang_isConstQualifiedType", [Type], bool),
@@ -3909,31 +3896,27 @@ def write_main_file_to_stdout(self):
     ),
     ("clang_visitChildren", [Cursor, cursor_visit_callback, py_object], c_uint),
     ("clang_Cursor_getNumArguments", [Cursor], c_int),
-    ("clang_Cursor_getArgument", [Cursor, c_uint], Cursor, Cursor.from_result),
+    ("clang_Cursor_getArgument", [Cursor, c_uint], Cursor),
     ("clang_Cursor_getNumTemplateArguments", [Cursor], c_int),
-    (
-        "clang_Cursor_getTemplateArgumentKind",
-        [Cursor, c_uint],
-        TemplateArgumentKind.from_id,
-    ),
-    ("clang_Cursor_getTemplateArgumentType", [Cursor, c_uint], Type, Type.from_result),
+    ("clang_Cursor_getTemplateArgumentKind", [Cursor, c_uint], c_uint),
+    ("clang_Cursor_getTemplateArgumentType", [Cursor, c_uint], Type),
     ("clang_Cursor_getTemplateArgumentValue", [Cursor, c_uint], c_longlong),
     ("clang_Cursor_getTemplateArgumentUnsignedValue", [Cursor, c_uint], c_ulonglong),
     ("clang_Cursor_isAnonymous", [Cursor], bool),
     ("clang_Cursor_isBitField", [Cursor], bool),
     ("clang_Cursor_getBinaryOpcode", [Cursor], c_int),
-    ("clang_Cursor_getBriefCommentText", [Cursor], _CXString, _CXString.from_result),
-    ("clang_Cursor_getRawCommentText", [Cursor], _CXString, _CXString.from_result),
+    ("clang_Cursor_getBriefCommentText", [Cursor], _CXString),
+    ("clang_Cursor_getRawCommentText", [Cursor], _CXString),
     ("clang_Cursor_getOffsetOfField", [Cursor], c_longlong),
     ("clang_Location_isInSystemHeader", [SourceLocation], bool),
     ("clang_Type_getAlignOf", [Type], c_longlong),
-    ("clang_Type_getClassType", [Type], Type, Type.from_result),
+    ("clang_Type_getClassType", [Type], Type),
     ("clang_Type_getNumTemplateArguments", [Type], c_int),
-    ("clang_Type_getTemplateArgumentAsType", [Type, c_uint], Type, Type.from_result),
+    ("clang_Type_getTemplateArgumentAsType", [Type, c_uint], Type),
     ("clang_Type_getOffsetOf", [Type, c_interop_string], c_longlong),
     ("clang_Type_getSizeOf", [Type], c_longlong),
     ("clang_Type_getCXXRefQualifier", [Type], c_uint),
-    ("clang_Type_getNamedType", [Type], Type, Type.from_result),
+    ("clang_Type_getNamedType", [Type], Type),
     ("clang_Type_visitFields", [Type, fields_visit_callback, py_object], c_uint),
 ]
 
diff --git a/clang/bindings/python/tests/cindex/test_cursor.py b/clang/bindings/python/tests/cindex/test_cursor.py
index 7476947bde2ea..77d8ca415708f 100644
--- a/clang/bindings/python/tests/cindex/test_cursor.py
+++ b/clang/bindings/python/tests/cindex/test_cursor.py
@@ -14,6 +14,7 @@
 from clang.cindex import TranslationUnit
 from clang.cindex import TypeKind
 from clang.cindex import BinaryOperator
+from clang.cindex import StorageClass
 from .util import get_cursor
 from .util import get_cursors
 from .util import get_tu
@@ -279,6 +280,90 @@ def test_is_default_method(self):
         self.assertTrue(xc.is_default_method())
         self.assertFalse(yc.is_default_method())
 
+    def test_is_deleted_method(self):
+        source = "class X { X() = delete; }; class Y { Y(); };"
+        tu = get_tu(source, lang="cpp")
+
+        xs = get_cursors(tu, "X")
+        ys = get_cursors(tu, "Y")
+
+        self.assertEqual(len(xs), 2)
+        self.assertEqual(len(ys), 2)
+
+        xc = xs[1]
+        yc = ys[1]
+
+        self.assertTrue(xc.is_deleted_method())
+        self.assertFalse(yc.is_deleted_method())
+
+    def test_is_copy_assignment_operator_method(self):
+        source_with_copy_assignment_operators = """
+        struct Foo {
+           // Those are copy-assignment operators
+           bool operator=(const Foo&);
+           bool operator=(Foo&);
+           Foo operator=(Foo);
+           bool operator=(volatile Foo&);
+           bool operator=(const volatile Foo&);
+
+        // Positive-check that the recognition works for templated classes too
+        template <typename T>
+        class Bar {
+            bool operator=(const Bar&);
+            Bar operator=(const Bar);
+            bool operator=(Bar<T>&);
+            bool operator=(volatile Bar&);
+            bool operator=(const volatile Bar<T>&);
+        };
+        """
+        source_without_copy_assignment_operators = """
+        struct Foo {
+            // Those are not copy-assignment operators
+            template<typename T>
+            bool operator=(const T&);
+            bool operator=(const bool&);
+            bool operator=(char&);
+            bool operator=(volatile unsigned int&);
+            bool operator=(const volatile unsigned char&);
+            bool operator=(int);
+            bool operator=(Foo&&);
+        };
+        """
+        tu_with_copy_assignment_operators = get_tu(
+            source_with_copy_assignment_operators, lang="cpp"
+        )
+        tu_without_copy_assignment_operators = get_tu(
+            source_without_copy_assignment_operators, lang="cpp"
+        )
+
+        copy_assignment_operators_cursors = get_cursors(
+            tu_with_copy_assignment_operators, "operator="
+        )
+        non_copy_assignment_operators_cursors = get_cursors(
+            tu_without_copy_assignment_operators, "operator="
+        )
+
+        self.assertEqual(len(copy_assignment_operators_cursors), 10)
+        self.assertTrue(len(non_copy_assignment_operators_cursors), 9)
+
+        self.assertTrue(
+            all(
+                [
+                    cursor.is_copy_assignment_operator_method()
+                    for cursor in copy_assignment_operators_cursors
+                ]
+            )
+        )
+
+        self.assertFalse(
+            any(
+                [
+                    cursor.is_copy_assignment_operator_method()
+                    for cursor in non_copy_assignment_operators_cursors
+                ]
+            )
+        )
+
     def test_is_move_assignment_operator_method(self):
         """Ensure Cursor.is_move_assignment_operator_method works."""
         source_with_move_assignment_operators = """
@@ -482,6 +567,41 @@ def test_is_scoped_enum(self):
         self.assertFalse(regular_enum.is_scoped_enum())
         self.assertTrue(scoped_enum.is_scoped_enum())
 
+    def test_get_definition(self):
+        """Ensure Cursor.get_definition works."""
+        tu = get_tu(
+            """
+class A {
+    constexpr static int f(){return 3;}
+};
+struct B {
+    int b = A::f();
+};
+""",
+            lang="cpp",
+        )
+        curs = get_cursors(tu, "f")
+        self.assertEqual(len(curs), 4)
+        self.assertEqual(curs[0].kind, CursorKind.CXX_METHOD)
+        self.assertEqual(curs[1].get_definition(), curs[0])
+        self.assertEqual(curs[2].get_definition(), curs[0])
+        self.assertEqual(curs[3].get_definition(), curs[0])
+
+    def test_get_usr(self):
+        """Ensure Cursor.get_usr works."""
+        tu = get_tu(
+            """
+int add(int, int);
+int add(int a, int b) { return a + b; }
+int add(float a, float b) { return a + b; }
+""",
+            lang="cpp",
+        )
+        curs = get_cursors(tu, "add")
+        self.assertEqual(len(curs), 3)
+        self.assertEqual(curs[0].get_usr(), curs[1].get_usr())
+        self.assertNotEqual(curs[0].get_usr(), curs[2].get_usr())
+
     def test_underlying_type(self):
         tu = get_tu("typedef int foo;")
         typedef = get_cursor(tu, "foo")
@@ -570,6 +690,23 @@ def test_enum_values_cpp(self):
         self.assertEqual(ham.kind, CursorKind.ENUM_CONSTANT_DECL)
         self.assertEqual(ham.enum_value, 0x10000000000)
 
+    def test_enum_values_unsigned(self):
+        tu = get_tu("enum TEST : unsigned char { SPAM=0, HAM = 200};", lang="cpp")
+        enum = get_cursor(tu, "TEST")
+        self.assertIsNotNone(enum)
+
+        self.assertEqual(enum.kind, CursorKind.ENUM_DECL)
+
+        enum_constants = list(enum.get_children())
+        self.assertEqual(len(enum_constants), 2)
+
+        spam, ham = enum_constants
+
+        self.assertEqual(spam.kind, CursorKind.ENUM_CONSTANT_DECL)
+        self.assertEqual(spam.enum_value, 0)
+        self.assertEqual(ham.kind, CursorKind.ENUM_CONSTANT_DECL)
+        self.assertEqual(ham.enum_value, 200)
+
     def test_annotation_attribute(self):
         tu = get_tu(
             'int foo (void) __attribute__ ((annotate("here be annotation attribute")));'
@@ -625,6 +762,25 @@ def test_result_type_objc_method_decl(self):
         self.assertEqual(cursor.kind, CursorKind.OBJC_INSTANCE_METHOD_DECL)
         self.assertEqual(result_type.kind, TypeKind.VOID)
 
+    def test_storage_class(self):
+        tu = get_tu(
+            """
+extern int ex;
+register int reg;
+int count(int a, int b){
+    static int counter = 0;
+    return 0;
+}
+""",
+            lang="cpp",
+        )
+        cursor = get_cursor(tu, "ex")
+        self.assertEqual(cursor.storage_class, StorageClass.EXTERN)
+        cursor = get_cursor(tu, "counter")
+        self.assertEqual(cursor.storage_class, StorageClass.STATIC)
+        cursor = get_cursor(tu, "reg")
+        self.assertEqual(cursor.storage_class, StorageClass.REGISTER)
+
     def test_availability(self):
         tu = get_tu("class A { A(A const&) = delete; };", lang="cpp")
 
@@ -681,6 +837,23 @@ def test_get_token_cursor(self):
         r_cursor = t_cursor.referenced  # should not raise an exception
         self.assertEqual(r_cursor.kind, CursorKind.CLASS_DECL)
 
+    def test_get_field_offsetof(self):
+        tu = get_tu(
+            "struct myStruct {int a; char b; char c; short d; char e;};", lang="cpp"
+        )
+        c1 = get_cursor(tu, "myStruct")
+        c2 = get_cursor(tu, "a")
+        c3 = get_cursor(tu, "b")
+        c4 = get_cursor(tu, "c")
+        c5 = get_cursor(tu, "d")
+        c6 = get_cursor(tu, "e")
+        self.assertEqual(c1.get_field_offsetof(), -1)
+        self.assertEqual(c2.get_field_offsetof(), 0)
+        self.assertEqual(c3.get_field_offsetof(), 32)
+        self.assertEqual(c4.get_field_offsetof(), 40)
+        self.assertEqual(c5.get_field_offsetof(), 48)
+        self.assertEqual(c6.get_field_offsetof(), 64)
+
     def test_get_arguments(self):
         tu = get_tu("void foo(int i, int j);")
         foo = get_cursor(tu, "foo")
@@ -799,3 +972,13 @@ def test_binop(self):
         for op, typ in operators.items():
             c = get_cursor(tu, op)
             assert c.binary_operator == typ
+
+    def test_from_result_null(self):
+        tu = get_tu("int a = 1+2;", lang="cpp")
+        op = next(next(tu.cursor.get_children()).get_children())
+        self.assertEqual(op.kind, CursorKind.BINARY_OPERATOR)
+        self.assertEqual(op.get_definition(), None)
+
+    def test_from_cursor_result_null(self):
+        tu = get_tu("")
+        self.assertEqual(tu.cursor.semantic_parent, None)
diff --git a/clang/bindings/python/tests/cindex/test_diagnostics.py b/clang/bindings/python/tests/cindex/test_diagnostics.py
index 57c41baaa2541..041083d12c7f1 100644
--- a/clang/bindings/python/tests/cindex/test_diagnostics.py
+++ b/clang/bindings/python/tests/cindex/test_diagnostics.py
@@ -46,6 +46,8 @@ def test_diagnostic_fixit(self):
         self.assertEqual(tu.diagnostics[0].location.column, 26)
         self.assertRegex(tu.diagnostics[0].spelling, "use of GNU old-style.*")
         self.assertEqual(len(tu.diagnostics[0].fixits), 1)
+        with self.assertRaises(IndexError):
+            tu.diagnostics[0].fixits[1]
         self.assertEqual(tu.diagnostics[0].fixits[0].range.start.line, 1)
         self.assertEqual(tu.diagnostics[0].fixits[0].range.start.column, 26)
         self.assertEqual(tu.diagnostics[0].fixits[0].range.end.line, 1)
@@ -97,6 +99,8 @@ def test_diagnostic_children(self):
 
         children = d.children
         self.assertEqual(len(children), 1)
+        with self.assertRaises(IndexError):
+            children[1]
         self.assertEqual(children[0].severity, Diagnostic.Note)
         self.assertRegex(children[0].spelling, ".*declared here")
         self.assertEqual(children[0].location.line, 1)
@@ -111,3 +115,16 @@ def test_diagnostic_string_repr(self):
             repr(d),
             "<Diagnostic severity 3, location <SourceLocation file 't.c', line 1, column 26>, spelling \"expected ';' after struct\">",
         )
+
+    def test_diagnostic_string_format(self):
+        tu = get_tu("struct MissingSemicolon{}")
+        self.assertEqual(len(tu.diagnostics), 1)
+        d = tu.diagnostics[0]
+
+        self.assertEqual(str(d), "t.c:1:26: error: expected ';' after struct")
+        self.assertEqual(
+            d.format(0b111111),
+            "t.c:1:26: error: expected ';' after struct [3, Parse Issue]",
+        )
+        with self.assertRaises(ValueError):
+            d.format(0b1000000)
diff --git a/clang/bindings/python/tests/cindex/test_type.py b/clang/bindings/python/tests/cindex/test_type.py
index 1dd8db0e3e814..928a9794e4213 100644
--- a/clang/bindings/python/tests/cindex/test_type.py
+++ b/clang/bindings/python/tests/cindex/test_type.py
@@ -10,7 +10,9 @@
 from clang.cindex import CursorKind
 from clang.cindex import TranslationUnit
 from clang.cindex import TypeKind
+from clang.cindex import RefQualifierKind
 from .util import get_cursor
+from .util import get_cursors
 from .util import get_tu
 
 
@@ -308,10 +310,10 @@ def test_element_type(self):
     def test_invalid_element_type(self):
         """Ensure Type.element_type raises if type doesn't have elements."""
         tu = get_tu("int i;")
-        i = get_cursor(tu, "i")
-        self.assertIsNotNone(i)
-        with self.assertRaises(Exception):
-            i.element_type
+        ty = get_cursor(tu, "i").type
+        with self.assertRaises(Exception) as ctx:
+            ty.element_type
+        self.assertEqual(str(ctx.exception), "Element type not available on this type.")
 
     def test_element_count(self):
         """Ensure Type.element_count works."""
@@ -357,6 +359,49 @@ def test_is_restrict_qualified(self):
         self.assertTrue(i.type.is_restrict_qualified())
         self.assertFalse(j.type.is_restrict_qualified())
 
+    def test_get_result(self):
+        tu = get_tu("void foo(); int bar(char, short);")
+        foo = get_cursor(tu, "foo")
+        bar = get_cursor(tu, "bar")
+        self.assertEqual(foo.type.get_result().spelling, "void")
+        self.assertEqual(bar.type.get_result().spelling, "int")
+
+    def test_get_class_type(self):
+        tu = get_tu(
+            """
+class myClass
+{
+   char *myAttr;
+};
+
+char *myClass::*pMyAttr = &myClass::myAttr;
+""",
+            lang="cpp",
+        )
+        cur = get_cursor(tu, "pMyAttr")
+        self.assertEqual(cur.type.get_class_type().spelling, "myClass")
+
+    def test_get_named_type(self):
+        tu = get_tu("using char_alias = char; char_alias xyz;", lang="cpp")
+        cur = get_cursor(tu, "xyz")
+        self.assertEqual(cur.type.get_named_type().spelling, "char_alias")
+
+    def test_get_ref_qualifier(self):
+        tu = get_tu(
+            """
+class A
+{
+	const int& getAttr() const &;
+	int getAttr() const &&;
+};
+""",
+            lang="cpp",
+        )
+        getters = get_cursors(tu, "getAttr")
+        self.assertEqual(len(getters), 2)
+        self.assertEqual(getters[0].type.get_ref_qualifier(), RefQualifierKind.LVALUE)
+        self.assertEqual(getters[1].type.get_ref_qualifier(), RefQualifierKind.RVALUE)
+
     def test_record_layout(self):
         """Ensure Cursor.type.get_size, Cursor.type.get_align and
         Cursor.type.get_offset works."""
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index a16b9c44ef0ea..baf39befd796a 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -586,28 +586,36 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
   #pragma omp declare simd
   int min();
-attr()
-  matches "nodiscard", "nonnull", "noinline", and the whole "#pragma" line.
+
+The matcher attr()
+matches nodiscard, nonnull, noinline, and
+declare simd.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;</td><td class="name" onclick="toggle('cxxBaseSpecifier0')"><a name="cxxBaseSpecifier0Anchor">cxxBaseSpecifier</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxBaseSpecifier0"><pre>Matches class bases.
 
-Examples matches public virtual B.
+Given
   class B {};
   class C : public virtual B {};
+
+The matcher cxxRecordDecl(hasDirectBase(cxxBaseSpecifier()))
+matches C.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;</td><td class="name" onclick="toggle('cxxCtorInitializer0')"><a name="cxxCtorInitializer0Anchor">cxxCtorInitializer</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxCtorInitializer0"><pre>Matches constructor initializers.
 
-Examples matches i(42).
+Given
   class C {
     C() : i(42) {}
     int i;
   };
+
+The matcher cxxCtorInitializer()
+matches i(42).
 </pre></td></tr>
 
 
@@ -619,17 +627,22 @@ <h2 id="decl-matchers">Node Matchers</h2>
   public:
     int a;
   };
-accessSpecDecl()
-  matches 'public:'
+
+The matcher accessSpecDecl()
+matches public:.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('bindingDecl0')"><a name="bindingDecl0Anchor">bindingDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BindingDecl.html">BindingDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="bindingDecl0"><pre>Matches binding declarations
-Example matches foo and bar
-(matcher = bindingDecl()
 
-  auto [foo, bar] = std::make_pair{42, 42};
+Given
+  struct pair { int x; int y; };
+  pair make(int, int);
+  auto [foo, bar] = make(42, 42);
+
+The matcher bindingDecl()
+matches foo and bar.
 </pre></td></tr>
 
 
@@ -642,14 +655,18 @@ <h2 id="decl-matchers">Node Matchers</h2>
   myFunc(^(int p) {
     printf("%d", p);
   })
+
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('classTemplateDecl0')"><a name="classTemplateDecl0Anchor">classTemplateDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ClassTemplateDecl.html">ClassTemplateDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="classTemplateDecl0"><pre>Matches C++ class template declarations.
 
-Example matches Z
+Given
   template&lt;class T&gt; class Z {};
+
+The matcher classTemplateDecl()
+matches Z.
 </pre></td></tr>
 
 
@@ -660,13 +677,14 @@ <h2 id="decl-matchers">Node Matchers</h2>
   template&lt;class T1, class T2, int I&gt;
   class A {};
 
-  template&lt;class T, int I&gt;
-  class A&lt;T, T*, I&gt; {};
+  template&lt;class T, int I&gt; class A&lt;T, T*, I&gt; {};
 
   template&lt;&gt;
   class A&lt;int, int, 1&gt; {};
-classTemplatePartialSpecializationDecl()
-  matches the specialization A&lt;T,T*,I&gt; but not A&lt;int,int,1&gt;
+
+The matcher classTemplatePartialSpecializationDecl()
+matches template&lt;class T, int I&gt; class A&lt;T, T*, I&gt; {},
+but does not match A&lt;int, int, 1&gt;.
 </pre></td></tr>
 
 
@@ -677,87 +695,128 @@ <h2 id="decl-matchers">Node Matchers</h2>
   template&lt;typename T&gt; class A {};
   template&lt;&gt; class A&lt;double&gt; {};
   A&lt;int&gt; a;
-classTemplateSpecializationDecl()
-  matches the specializations A&lt;int&gt; and A&lt;double&gt;
+
+The matcher classTemplateSpecializationDecl()
+matches class A&lt;int&gt;
+and class A&lt;double&gt;.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('conceptDecl0')"><a name="conceptDecl0Anchor">conceptDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ConceptDecl.html">ConceptDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="conceptDecl0"><pre>Matches concept declarations.
 
-Example matches integral
-  template&lt;typename T&gt;
-  concept integral = std::is_integral_v&lt;T&gt;;
+Given
+  template&lt;typename T&gt; concept my_concept = true;
+
+
+The matcher conceptDecl()
+matches template&lt;typename T&gt;
+concept my_concept = true.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxConstructorDecl0')"><a name="cxxConstructorDecl0Anchor">cxxConstructorDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructorDecl.html">CXXConstructorDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxConstructorDecl0"><pre>Matches C++ constructor declarations.
 
-Example matches Foo::Foo() and Foo::Foo(int)
+Given
   class Foo {
    public:
     Foo();
     Foo(int);
     int DoSomething();
   };
+
+  struct Bar {};
+
+
+The matcher cxxConstructorDecl()
+matches Foo() and Foo(int).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxConversionDecl0')"><a name="cxxConversionDecl0Anchor">cxxConversionDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConversionDecl.html">CXXConversionDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxConversionDecl0"><pre>Matches conversion operator declarations.
 
-Example matches the operator.
+Given
   class X { operator int() const; };
+
+
+The matcher cxxConversionDecl()
+matches operator int() const.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxDeductionGuideDecl0')"><a name="cxxDeductionGuideDecl0Anchor">cxxDeductionGuideDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXDeductionGuideDecl.html">CXXDeductionGuideDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxDeductionGuideDecl0"><pre>Matches user-defined and implicitly generated deduction guide.
 
-Example matches the deduction guide.
+Given
   template&lt;typename T&gt;
-  class X { X(int) };
+  class X { X(int); };
   X(int) -&gt; X&lt;int&gt;;
+
+
+The matcher cxxDeductionGuideDecl()
+matches the written deduction guide
+auto (int) -&gt; X&lt;int&gt;,
+the implicit copy deduction guide auto (int) -&gt; X&lt;T&gt;
+and the implicitly declared deduction guide
+auto (X&lt;T&gt;) -&gt; X&lt;T&gt;.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxDestructorDecl0')"><a name="cxxDestructorDecl0Anchor">cxxDestructorDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXDestructorDecl.html">CXXDestructorDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxDestructorDecl0"><pre>Matches explicit C++ destructor declarations.
 
-Example matches Foo::~Foo()
+Given
   class Foo {
    public:
     virtual ~Foo();
   };
+
+  struct Bar {};
+
+
+The matcher cxxDestructorDecl()
+matches virtual ~Foo().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxMethodDecl0')"><a name="cxxMethodDecl0Anchor">cxxMethodDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxMethodDecl0"><pre>Matches method declarations.
 
-Example matches y
+Given
   class X { void y(); };
+
+
+The matcher cxxMethodDecl()
+matches void y().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('cxxRecordDecl0')"><a name="cxxRecordDecl0Anchor">cxxRecordDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxRecordDecl0"><pre>Matches C++ class declarations.
 
-Example matches X, Z
+Given
   class X;
   template&lt;class T&gt; class Z {};
+
+The matcher cxxRecordDecl()
+matches X and Z.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('decl0')"><a name="decl0Anchor">decl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="decl0"><pre>Matches declarations.
 
-Examples matches X, C, and the friend declaration inside C;
+Given
   void X();
   class C {
-    friend X;
+    friend void X();
   };
+
+The matcher decl()
+matches void X(), C
+and friend void X().
 </pre></td></tr>
 
 
@@ -767,40 +826,49 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   class X { int y; };
-declaratorDecl()
-  matches int y.
+
+The matcher declaratorDecl()
+matches int y.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('decompositionDecl0')"><a name="decompositionDecl0Anchor">decompositionDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecompositionDecl.html">DecompositionDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="decompositionDecl0"><pre>Matches decomposition-declarations.
 
-Examples matches the declaration node with foo and bar, but not
-number.
-(matcher = declStmt(has(decompositionDecl())))
-
+Given
+  struct pair { int x; int y; };
+  pair make(int, int);
   int number = 42;
-  auto [foo, bar] = std::make_pair{42, 42};
+  auto [foo, bar] = make(42, 42);
+
+The matcher decompositionDecl()
+matches auto [foo, bar] = make(42, 42),
+but does not match number.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('enumConstantDecl0')"><a name="enumConstantDecl0Anchor">enumConstantDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1EnumConstantDecl.html">EnumConstantDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="enumConstantDecl0"><pre>Matches enum constants.
 
-Example matches A, B, C
+Given
   enum X {
     A, B, C
   };
+The matcher enumConstantDecl()
+matches A, B and C.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('enumDecl0')"><a name="enumDecl0Anchor">enumDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1EnumDecl.html">EnumDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="enumDecl0"><pre>Matches enum declarations.
 
-Example matches X
+Given
   enum X {
     A, B, C
   };
+
+The matcher enumDecl()
+matches the enum X.
 </pre></td></tr>
 
 
@@ -808,9 +876,14 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="fieldDecl0"><pre>Matches field declarations.
 
 Given
-  class X { int m; };
-fieldDecl()
-  matches 'm'.
+  int a;
+  struct Foo {
+    int x;
+  };
+  void bar(int val);
+
+The matcher fieldDecl()
+matches int x.
 </pre></td></tr>
 
 
@@ -819,16 +892,20 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   class X { friend void foo(); };
-friendDecl()
-  matches 'friend void foo()'.
+
+The matcher friendDecl()
+matches friend void foo().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('functionDecl0')"><a name="functionDecl0Anchor">functionDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="functionDecl0"><pre>Matches function declarations.
 
-Example matches f
+Given
   void f();
+
+The matcher functionDecl()
+matches void f().
 </pre></td></tr>
 
 
@@ -837,6 +914,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Example matches f
   template&lt;class T&gt; void f(T t) {}
+
+
+The matcher functionTemplateDecl()
+matches template&lt;class T&gt; void f(T t) {}.
 </pre></td></tr>
 
 
@@ -845,8 +926,8 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   struct X { struct { int a; }; };
-indirectFieldDecl()
-  matches 'a'.
+The matcher indirectFieldDecl()
+matches a.
 </pre></td></tr>
 
 
@@ -854,10 +935,13 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="labelDecl0"><pre>Matches a declaration of label.
 
 Given
-  goto FOO;
-  FOO: bar();
-labelDecl()
-  matches 'FOO:'
+  void bar();
+  void foo() {
+    goto FOO;
+    FOO: bar();
+  }
+The matcher labelDecl()
+matches FOO: bar().
 </pre></td></tr>
 
 
@@ -866,8 +950,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   extern "C" {}
-linkageSpecDecl()
-  matches "extern "C" {}"
+
+The matcher linkageSpecDecl()
+matches extern "C" {}.
 </pre></td></tr>
 
 
@@ -875,12 +960,18 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="namedDecl0"><pre>Matches a declaration of anything that could have a name.
 
 Example matches X, S, the anonymous union type, i, and U;
+Given
   typedef int X;
   struct S {
     union {
       int i;
     } U;
   };
+The matcher namedDecl()
+matches typedef int X, S, int i
+ and U,
+with S matching twice in C++.
+Once for the injected class name and once for the declaration itself.
 </pre></td></tr>
 
 
@@ -890,8 +981,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   namespace test {}
   namespace alias = ::test;
-namespaceAliasDecl()
-  matches "namespace alias" but not "namespace test"
+
+The matcher namespaceAliasDecl()
+matches alias,
+but does not match test.
 </pre></td></tr>
 
 
@@ -901,8 +994,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   namespace {}
   namespace test {}
-namespaceDecl()
-  matches "namespace {}" and "namespace test {}"
+
+The matcher namespaceDecl()
+matches namespace {} and namespace test {}.
 </pre></td></tr>
 
 
@@ -911,8 +1005,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   template &lt;typename T, int N&gt; struct C {};
-nonTypeTemplateParmDecl()
-  matches 'N', but not 'T'.
+
+The matcher nonTypeTemplateParmDecl()
+matches int N,
+but does not match typename T.
 </pre></td></tr>
 
 
@@ -922,6 +1018,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches Foo (Additions)
   @interface Foo (Additions)
   @end
+
 </pre></td></tr>
 
 
@@ -931,6 +1028,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches Foo (Additions)
   @implementation Foo (Additions)
   @end
+
 </pre></td></tr>
 
 
@@ -940,6 +1038,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches Foo
   @implementation Foo
   @end
+
 </pre></td></tr>
 
 
@@ -949,6 +1048,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches Foo
   @interface Foo
   @end
+
 </pre></td></tr>
 
 
@@ -960,6 +1060,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
     BOOL _enabled;
   }
   @end
+
 </pre></td></tr>
 
 
@@ -974,6 +1075,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
   @implementation Foo
   - (void)method {}
   @end
+
 </pre></td></tr>
 
 
@@ -984,6 +1086,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
   @interface Foo
   @property BOOL enabled;
   @end
+
 </pre></td></tr>
 
 
@@ -993,6 +1096,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches FooDelegate
   @protocol FooDelegate
   @end
+
 </pre></td></tr>
 
 
@@ -1001,48 +1105,58 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   void f(int x);
-parmVarDecl()
-  matches int x.
+The matcher parmVarDecl()
+matches int x.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('recordDecl0')"><a name="recordDecl0Anchor">recordDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1RecordDecl.html">RecordDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="recordDecl0"><pre>Matches class, struct, and union declarations.
 
-Example matches X, Z, U, and S
+Given
   class X;
   template&lt;class T&gt; class Z {};
   struct S {};
   union U {};
+
+The matcher recordDecl()
+matches X, Z,
+S and U.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('staticAssertDecl0')"><a name="staticAssertDecl0Anchor">staticAssertDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1StaticAssertDecl.html">StaticAssertDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="staticAssertDecl0"><pre>Matches a C++ static_assert declaration.
 
-Example:
-  staticAssertDecl()
-matches
-  static_assert(sizeof(S) == sizeof(int))
-in
+Given
   struct S {
     int x;
   };
   static_assert(sizeof(S) == sizeof(int));
+
+
+The matcher staticAssertDecl()
+matches static_assert(sizeof(S) == sizeof(int)).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('tagDecl0')"><a name="tagDecl0Anchor">tagDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="tagDecl0"><pre>Matches tag declarations.
 
-Example matches X, Z, U, S, E
+Given
   class X;
   template&lt;class T&gt; class Z {};
   struct S {};
   union U {};
-  enum E {
-    A, B, C
-  };
+  enum E { A, B, C };
+
+
+The matcher tagDecl()
+matches class X, class Z {}, the injected class name
+class Z, struct S {},
+the injected class name struct S, union U {},
+the injected class name union U
+and enum E { A, B, C }.
 </pre></td></tr>
 
 
@@ -1051,8 +1165,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   template &lt;template &lt;typename&gt; class Z, int N&gt; struct C {};
-templateTypeParmDecl()
-  matches 'Z', but not 'N'.
+
+The matcher templateTemplateParmDecl()
+matches template &lt;typename&gt; class Z,
+but does not match int N.
 </pre></td></tr>
 
 
@@ -1061,8 +1177,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   template &lt;typename T, int N&gt; struct C {};
-templateTypeParmDecl()
-  matches 'T', but not 'N'.
+
+The matcher templateTypeParmDecl()
+matches typename T,
+but does not int N.
 </pre></td></tr>
 
 
@@ -1072,10 +1190,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   int X;
   namespace NS {
-  int Y;
+    int Y;
   }  // namespace NS
-decl(hasDeclContext(translationUnitDecl()))
-  matches "int X", but not "int Y".
+
+The matcher namedDecl(hasDeclContext(translationUnitDecl()))
+matches X and NS,
+but does not match Y.
 </pre></td></tr>
 
 
@@ -1085,17 +1205,22 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   typedef int X;
   using Y = int;
-typeAliasDecl()
-  matches "using Y = int", but not "typedef int X"
+
+The matcher typeAliasDecl()
+matches using Y = int,
+but does not match typedef int X.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('typeAliasTemplateDecl0')"><a name="typeAliasTemplateDecl0Anchor">typeAliasTemplateDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeAliasTemplateDecl.html">TypeAliasTemplateDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="typeAliasTemplateDecl0"><pre>Matches type alias template declarations.
 
-typeAliasTemplateDecl() matches
-  template &lt;typename T&gt;
-  using Y = X&lt;T&gt;;
+Given
+  template &lt;typename T&gt; struct X {};
+  template &lt;typename T&gt; using Y = X&lt;T&gt;;
+
+The matcher typeAliasTemplateDecl()
+matches template &lt;typename T&gt; using Y = X&lt;T&gt;.
 </pre></td></tr>
 
 
@@ -1105,8 +1230,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   typedef int X;
   using Y = int;
-typedefDecl()
-  matches "typedef int X", but not "using Y = int"
+
+The matcher typedefDecl()
+matches typedef int X,
+but does not match using Y = int.
 </pre></td></tr>
 
 
@@ -1116,8 +1243,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   typedef int X;
   using Y = int;
-typedefNameDecl()
-  matches "typedef int X" and "using Y = int"
+
+The matcher typedefNameDecl()
+matches typedef int X and using Y = int.
 </pre></td></tr>
 
 
@@ -1133,8 +1261,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   struct S : private Base&lt;T&gt; {
     using typename Base&lt;T&gt;::Foo;
   };
-unresolvedUsingTypenameDecl()
-  matches using Base&lt;T&gt;::Foo </pre></td></tr>
+
+The matcher unresolvedUsingTypenameDecl()
+  matches using typename Base&lt;T&gt;::Foo
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('unresolvedUsingValueDecl0')"><a name="unresolvedUsingValueDecl0Anchor">unresolvedUsingValueDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnresolvedUsingValueDecl.html">UnresolvedUsingValueDecl</a>&gt;...</td></tr>
@@ -1145,8 +1275,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   class C : private X {
     using X::x;
   };
-unresolvedUsingValueDecl()
-  matches using X::x </pre></td></tr>
+
+The matcher unresolvedUsingValueDecl()
+  matches using X::x
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('usingDecl0')"><a name="usingDecl0Anchor">usingDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingDecl.html">UsingDecl</a>&gt;...</td></tr>
@@ -1155,8 +1287,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   namespace X { int x; }
   using X::x;
-usingDecl()
-  matches using X::x </pre></td></tr>
+
+The matcher usingDecl()
+  matches using X::x
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('usingDirectiveDecl0')"><a name="usingDirectiveDecl0Anchor">usingDirectiveDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingDirectiveDecl.html">UsingDirectiveDecl</a>&gt;...</td></tr>
@@ -1165,26 +1299,34 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   namespace X { int x; }
   using namespace X;
-usingDirectiveDecl()
-  matches using namespace X </pre></td></tr>
+
+The matcher usingDirectiveDecl()
+  matches using namespace X
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('usingEnumDecl0')"><a name="usingEnumDecl0Anchor">usingEnumDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingEnumDecl.html">UsingEnumDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="usingEnumDecl0"><pre>Matches using-enum declarations.
 
 Given
-  namespace X { enum x {...}; }
+  namespace X { enum x { val1, val2 }; }
   using enum X::x;
-usingEnumDecl()
-  matches using enum X::x </pre></td></tr>
+
+The matcher usingEnumDecl()
+  matches using enum X::x
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('valueDecl0')"><a name="valueDecl0Anchor">valueDecl</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="valueDecl0"><pre>Matches any value declaration.
 
-Example matches A, B, C and F
+Given
   enum X { A, B, C };
   void F();
+  int V = 0;
+The matcher valueDecl()
+matches A, B, C, void F()
+and int V = 0.
 </pre></td></tr>
 
 
@@ -1196,6 +1338,13 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Example matches a
   int a;
+  struct Foo {
+    int x;
+  };
+  void bar(int val);
+
+The matcher varDecl()
+matches int a and int val, but not int x.
 </pre></td></tr>
 
 
@@ -1208,13 +1357,29 @@ <h2 id="decl-matchers">Node Matchers</h2>
     auto f = [x](){};
     auto g = [x = 1](){};
   }
-In the matcher `lambdaExpr(hasAnyCapture(lambdaCapture()))`,
-`lambdaCapture()` matches `x` and `x=1`.
+
+The matcher
+lambdaExpr(hasAnyCapture(lambdaCapture().bind("capture"))),
+matches [x](){} and [x = 1](){},
+with lambdaCapture() matching
+x and x = 1.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifierLoc.html">NestedNameSpecifierLoc</a>&gt;</td><td class="name" onclick="toggle('nestedNameSpecifierLoc0')"><a name="nestedNameSpecifierLoc0Anchor">nestedNameSpecifierLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifierLoc.html">NestedNameSpecifierLoc</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="nestedNameSpecifierLoc0"><pre>Same as nestedNameSpecifier but matches NestedNameSpecifierLoc.
+
+Given
+  namespace ns {
+    struct A { static void f(); };
+    void A::f() {}
+    void g() { A::f(); }
+  }
+  ns::A a;
+
+
+The matcher nestedNameSpecifierLoc() matches
+A:: twice, and ns:: once.
 </pre></td></tr>
 
 
@@ -1228,8 +1393,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
     void g() { A::f(); }
   }
   ns::A a;
-nestedNameSpecifier()
-  matches "ns::" and both "A::"
+
+The matcher nestedNameSpecifier()
+matches ns and both A
 </pre></td></tr>
 
 
@@ -1237,20 +1403,38 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="ompDefaultClause0"><pre>Matches OpenMP ``default`` clause.
 
 Given
+  void foo() {
+    #pragma omp parallel default(none)
+      ;
+    #pragma omp parallel default(shared)
+      ;
+    #pragma omp parallel default(private)
+      ;
+    #pragma omp parallel default(firstprivate)
+      ;
+    #pragma omp parallel
+      ;
+  }
 
-  #pragma omp parallel default(none)
-  #pragma omp parallel default(shared)
-  #pragma omp parallel default(private)
-  #pragma omp parallel default(firstprivate)
-  #pragma omp parallel
 
-``ompDefaultClause()`` matches ``default(none)``, ``default(shared)``,
-`` default(private)`` and ``default(firstprivate)``
+The matcher
+ompExecutableDirective(hasAnyClause(ompDefaultClause())) matches
+#pragma omp parallel default(none),
+#pragma omp parallel default(shared),
+#pragma omp parallel default(private) and
+#pragma omp parallel default(firstprivate).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('qualType0')"><a name="qualType0Anchor">qualType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="qualType0"><pre>Matches QualTypes in the clang AST.
+
+Given
+  int a = 0;
+  const int b = 1;
+
+The matcher varDecl(hasType(qualType(isConstQualified())))
+matches const int b = 1, but not int a = 0.
 </pre></td></tr>
 
 
@@ -1258,34 +1442,39 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="addrLabelExpr0"><pre>Matches address of label statements (GNU extension).
 
 Given
+void bar();
+void foo() {
   FOO: bar();
   void *ptr = &amp;&amp;FOO;
-  goto *bar;
-addrLabelExpr()
-  matches '&amp;&amp;FOO'
+  goto *ptr;
+}
+The matcher addrLabelExpr()
+matches &amp;&amp;FOO
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('arrayInitIndexExpr0')"><a name="arrayInitIndexExpr0Anchor">arrayInitIndexExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArrayInitIndexExpr.html">ArrayInitIndexExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="arrayInitIndexExpr0"><pre>The arrayInitIndexExpr consists of two subexpressions: a common expression
-(the source array) that is evaluated once up-front, and a per-element initializer
-that runs once for each array element. Within the per-element initializer,
-the current index may be obtained via an ArrayInitIndexExpr.
+(the source array) that is evaluated once up-front, and a per-element
+initializer that runs once for each array element. Within the per-element
+initializer, the current index may be obtained via an ArrayInitIndexExpr.
 
 Given
-  void testStructBinding() {
+  void testStructuredBinding() {
     int a[2] = {1, 2};
     auto [x, y] = a;
   }
-arrayInitIndexExpr() matches the array index that implicitly iterates
-over the array `a` to copy each element to the anonymous array
-that backs the structured binding `[x, y]` elements of which are
-referred to by their aliases `x` and `y`.
+
+
+The matcher arrayInitIndexExpr() matches the array index
+that implicitly iterates over the array `a` to copy each element to the
+anonymous array that backs the structured binding.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('arrayInitLoopExpr0')"><a name="arrayInitLoopExpr0Anchor">arrayInitLoopExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArrayInitLoopExpr.html">ArrayInitLoopExpr</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="arrayInitLoopExpr0"><pre>Matches a loop initializing the elements of an array in a number of contexts:
+<tr><td colspan="4" class="doc" id="arrayInitLoopExpr0"><pre>Matches a loop initializing the elements of an array in a number of
+contexts:
  * in the implicit copy/move constructor for a class with an array member
  * when a lambda-expression captures an array by value
  * when a decomposition declaration decomposes an array
@@ -1293,13 +1482,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   void testLambdaCapture() {
     int a[10];
-    auto Lam1 = [a]() {
-      return;
-    };
+    [a]() {};
   }
-arrayInitLoopExpr() matches the implicit loop that initializes each element of
-the implicit array field inside the lambda object, that represents the array `a`
-captured by value.
+
+The matcher arrayInitLoopExpr() matches the implicit loop that
+initializes each element of the implicit array field inside the lambda
+object, that represents the array a captured by value.
 </pre></td></tr>
 
 
@@ -1307,26 +1495,34 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="arraySubscriptExpr0"><pre>Matches array subscript expressions.
 
 Given
-  int i = a[1];
-arraySubscriptExpr()
-  matches "a[1]"
+  void foo() {
+    int a[2] = {0, 1};
+    int i = a[1];
+  }
+The matcher arraySubscriptExpr()
+matches a[1].
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('asmStmt0')"><a name="asmStmt0Anchor">asmStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AsmStmt.html">AsmStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="asmStmt0"><pre>Matches asm statements.
 
+void foo() {
  int i = 100;
-  __asm("mov al, 2");
-asmStmt()
-  matches '__asm("mov al, 2")'
+  __asm("mov %al, 2");
+}
+The matcher asmStmt()
+matches __asm("mov %al, 2")
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('atomicExpr0')"><a name="atomicExpr0Anchor">atomicExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AtomicExpr.html">AtomicExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="atomicExpr0"><pre>Matches atomic builtins.
-Example matches __atomic_load_n(ptr, 1)
+
+Given
   void foo() { int *ptr; __atomic_load_n(ptr, 1); }
+
+The matcher atomicExpr() matches __atomic_load_n(ptr, 1).
 </pre></td></tr>
 
 
@@ -1337,24 +1533,35 @@ <h2 id="decl-matchers">Node Matchers</h2>
   @autoreleasepool {
     int x = 0;
   }
-autoreleasePoolStmt(stmt()) matches the declaration of "x"
-inside the autorelease pool.
+
+The matcher autoreleasePoolStmt(stmt()) matches the declaration of
+int x = 0 inside the autorelease pool.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('binaryConditionalOperator0')"><a name="binaryConditionalOperator0Anchor">binaryConditionalOperator</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryConditionalOperator.html">BinaryConditionalOperator</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="binaryConditionalOperator0"><pre>Matches binary conditional operator expressions (GNU extension).
 
-Example matches a ?: b
-  (a ?: b) + 42;
+Given
+  int f(int a, int b) {
+    return (a ?: b) + 42;
+  }
+
+The matcher binaryConditionalOperator() matches a ?: b.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('binaryOperator0')"><a name="binaryOperator0Anchor">binaryOperator</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="binaryOperator0"><pre>Matches binary operator expressions.
 
-Example matches a || b
-  !(a || b)
+Given
+  void foo(bool a, bool b) {
+    !(a || b);
+  }
+
+
+The matcher binaryOperator() matches a || b.
+
 See also the binaryOperation() matcher for more-general matching.
 </pre></td></tr>
 
@@ -1362,8 +1569,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('blockExpr0')"><a name="blockExpr0Anchor">blockExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockExpr.html">BlockExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="blockExpr0"><pre>Matches a reference to a block.
 
-Example: matches "^{}":
+Given
   void f() { ^{}(); }
+
+
+The matcher blockExpr() matches ^{}.
 </pre></td></tr>
 
 
@@ -1371,17 +1581,23 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="breakStmt0"><pre>Matches break statements.
 
 Given
+void foo() {
   while (true) { break; }
-breakStmt()
-  matches 'break'
+}
+
+The matcher breakStmt()
+matches break
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cStyleCastExpr0')"><a name="cStyleCastExpr0Anchor">cStyleCastExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CStyleCastExpr.html">CStyleCastExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cStyleCastExpr0"><pre>Matches a C-style cast expression.
 
-Example: Matches (int) 2.2f in
+Given
   int i = (int) 2.2f;
+
+The matcher cStyleCastExpr()
+matches (int) 2.2f.
 </pre></td></tr>
 
 
@@ -1389,9 +1605,16 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="callExpr0"><pre>Matches call expressions.
 
 Example matches x.y() and y()
-  X x;
-  x.y();
-  y();
+  struct X { void foo(); };
+  void bar();
+  void foobar() {
+    X x;
+    x.foo();
+    bar();
+  }
+
+The matcher callExpr()
+matches x.foo() and bar();
 </pre></td></tr>
 
 
@@ -1399,22 +1622,41 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="caseStmt0"><pre>Matches case statements inside switch statements.
 
 Given
+void foo(int a) {
   switch(a) { case 42: break; default: break; }
-caseStmt()
-  matches 'case 42:'.
+}
+The matcher caseStmt()
+matches case 42: break.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('castExpr0')"><a name="castExpr0Anchor">castExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CastExpr.html">CastExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="castExpr0"><pre>Matches any cast nodes of Clang's AST.
 
-Example: castExpr() matches each of the following:
-  (int) 3;
-  const_cast&lt;Expr *&gt;(SubExpr);
-  char c = 0;
-but does not match
-  int i = (0);
-  int k = 0;
+Given
+  struct S {};
+  const S* s;
+  S* s2 = const_cast&lt;S*&gt;(s);
+
+  const int val = 0;
+  char val0 = 1;
+  char val1 = (char)2;
+  char val2 = static_cast&lt;char&gt;(3);
+  int* val3 = reinterpret_cast&lt;int*&gt;(4);
+  char val4 = char(5);
+
+
+The matcher castExpr()
+matches
+const_cast&lt;S*&gt;(s) and the implicit l- to r-value cast for s,
+the implicit cast to char for the initializer 1,
+the c-style cast (char)2 and it's implicit cast to char
+(part of the c-style cast) 2,
+static_cast&lt;char&gt;(3) and it's implicit cast to char
+(part of the static_cast) 3,
+reinterpret_cast&lt;int*&gt;(4),
+char(5) and it's implicit cast to char
+(part of the functional cast) 5.
 </pre></td></tr>
 
 
@@ -1424,14 +1666,24 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Not matching Hex-encoded chars (e.g. 0x1234, which is a IntegerLiteral),
 though.
 
-Example matches 'a', L'a'
+Given
   char ch = 'a';
   wchar_t chw = L'a';
+
+
+The matcher characterLiteral() matches 'a' and
+L'a'.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('chooseExpr0')"><a name="chooseExpr0Anchor">chooseExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ChooseExpr.html">ChooseExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="chooseExpr0"><pre>Matches GNU __builtin_choose_expr.
+
+Given
+  void f() { (void)__builtin_choose_expr(1, 2, 3); }
+
+The matcher chooseExpr() matches
+__builtin_choose_expr(1, 2, 3).
 </pre></td></tr>
 
 
@@ -1439,9 +1691,45 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="coawaitExpr0"><pre>Matches co_await expressions.
 
 Given
-  co_await 1;
-coawaitExpr()
-  matches 'co_await 1'
+  namespace std {
+  template &lt;typename T = void&gt;
+  struct coroutine_handle {
+      static constexpr coroutine_handle from_address(void* addr) {
+        return {};
+      }
+  };
+
+  struct always_suspend {
+      bool await_ready() const noexcept;
+      bool await_resume() const noexcept;
+      template &lt;typename T&gt;
+      bool await_suspend(coroutine_handle&lt;T&gt;) const noexcept;
+  };
+
+  template &lt;typename T&gt;
+  struct coroutine_traits {
+      using promise_type = T::promise_type;
+  };
+  }  // namespace std
+
+  struct generator {
+      struct promise_type {
+          std::always_suspend yield_value(int&amp;&amp;);
+          std::always_suspend initial_suspend() const noexcept;
+          std::always_suspend final_suspend() const noexcept;
+          void return_void();
+          void unhandled_exception();
+          generator get_return_object();
+      };
+  };
+
+  std::always_suspend h();
+
+  generator g() { co_await h(); }
+
+The matcher
+coawaitExpr(has(callExpr(callee(functionDecl(hasName("h"))))))
+matches co_await h().
 </pre></td></tr>
 
 
@@ -1449,35 +1737,48 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="compoundLiteralExpr0"><pre>Matches compound (i.e. non-scalar) literals
 
 Example match: {1}, (1, 2)
-  int array[4] = {1};
-  vector int myvec = (vector int)(1, 2);
+  struct vector { int x; int y; };
+  struct vector myvec = (struct vector){ 1, 2 };
+
+The matcher compoundLiteralExpr()
+matches (struct vector){ 1, 2 }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('compoundStmt0')"><a name="compoundStmt0Anchor">compoundStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundStmt.html">CompoundStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="compoundStmt0"><pre>Matches compound statements.
 
-Example matches '{}' and '{{}}' in 'for (;;) {{}}'
-  for (;;) {{}}
+Given
+void foo() { for (;;) {{}} }
+
+The matcher compoundStmt() matches
+{ for (;;) {{}} }, {{}} and {}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('conditionalOperator0')"><a name="conditionalOperator0Anchor">conditionalOperator</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ConditionalOperator.html">ConditionalOperator</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="conditionalOperator0"><pre>Matches conditional operator expressions.
 
-Example matches a ? b : c
-  (a ? b : c) + 42
+Given
+  int f(int a, int b, int c) {
+    return (a ? b : c) + 42;
+  }
+
+The matcher conditionalOperator() matches a ? b : c.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('constantExpr0')"><a name="constantExpr0Anchor">constantExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ConstantExpr.html">ConstantExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="constantExpr0"><pre>Matches a constant expression wrapper.
 
-Example matches the constant in the case statement:
-    (matcher = constantExpr())
-  switch (a) {
-  case 37: break;
+Given
+  void f(int a) {
+    switch (a) {
+      case 37: break;
+    }
   }
+
+The matcher constantExpr() matches 37.
 </pre></td></tr>
 
 
@@ -1485,14 +1786,26 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="continueStmt0"><pre>Matches continue statements.
 
 Given
+void foo() {
   while (true) { continue; }
-continueStmt()
-  matches 'continue'
+}
+
+The matcher continueStmt()
+matches continue
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('convertVectorExpr0')"><a name="convertVectorExpr0Anchor">convertVectorExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ConvertVectorExpr.html">ConvertVectorExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="convertVectorExpr0"><pre>Matches builtin function __builtin_convertvector.
+
+Given
+  typedef double vector4double __attribute__((__vector_size__(32)));
+  typedef float  vector4float  __attribute__((__vector_size__(16)));
+  vector4float vf;
+  void f() { (void)__builtin_convertvector(vf, vector4double); }
+
+The matcher convertVectorExpr() matches
+__builtin_convertvector(vf, vector4double).
 </pre></td></tr>
 
 
@@ -1500,19 +1813,85 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="coreturnStmt0"><pre>Matches co_return statements.
 
 Given
-  while (true) { co_return; }
-coreturnStmt()
-  matches 'co_return'
+  namespace std {
+  template &lt;typename T = void&gt;
+  struct coroutine_handle {
+      static constexpr coroutine_handle from_address(void* addr) {
+        return {};
+      }
+  };
+
+  struct always_suspend {
+      bool await_ready() const noexcept;
+      bool await_resume() const noexcept;
+      template &lt;typename T&gt;
+      bool await_suspend(coroutine_handle&lt;T&gt;) const noexcept;
+  };
+
+  template &lt;typename T&gt;
+  struct coroutine_traits {
+      using promise_type = T::promise_type;
+  };
+  }  // namespace std
+
+  struct generator {
+      struct promise_type {
+          void return_value(int v);
+          std::always_suspend yield_value(int&amp;&amp;);
+          std::always_suspend initial_suspend() const noexcept;
+          std::always_suspend final_suspend() const noexcept;
+          void unhandled_exception();
+          generator get_return_object();
+      };
+  };
+
+  generator f() {
+      co_return 10;
+  }
+
+
+The matcher coreturnStmt(has(integerLiteral()))
+matches co_return 10
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('coroutineBodyStmt0')"><a name="coroutineBodyStmt0Anchor">coroutineBodyStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CoroutineBodyStmt.html">CoroutineBodyStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="coroutineBodyStmt0"><pre>Matches coroutine body statements.
 
-coroutineBodyStmt() matches the coroutine below
-  generator&lt;int&gt; gen() {
-    co_return;
-  }
+Given
+  namespace std {
+  template &lt;typename T = void&gt;
+  struct coroutine_handle {
+      static constexpr coroutine_handle from_address(void* addr) {
+        return {};
+      }
+  };
+
+  struct suspend_always {
+      bool await_ready() const noexcept;
+      bool await_resume() const noexcept;
+      template &lt;typename T&gt;
+      bool await_suspend(coroutine_handle&lt;T&gt;) const noexcept;
+  };
+
+  template &lt;typename...&gt;
+  struct coroutine_traits {
+      struct promise_type {
+          std::suspend_always initial_suspend() const noexcept;
+          std::suspend_always final_suspend() const noexcept;
+          void return_void();
+          void unhandled_exception();
+          coroutine_traits get_return_object();
+      };
+  };
+  }  // namespace std
+
+  void f() { while (true) { co_return; } }
+
+
+
+The matcher coroutineBodyStmt() matches
+{ while (true) { co_return; } }.
 </pre></td></tr>
 
 
@@ -1520,27 +1899,77 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="coyieldExpr0"><pre>Matches co_yield expressions.
 
 Given
-  co_yield 1;
-coyieldExpr()
-  matches 'co_yield 1'
+  namespace std {
+  template &lt;typename T = void&gt;
+  struct coroutine_handle {
+      static constexpr coroutine_handle from_address(void* addr) {
+        return {};
+      }
+  };
+
+  struct always_suspend {
+      bool await_ready() const noexcept;
+      bool await_resume() const noexcept;
+      template &lt;typename T&gt;
+      bool await_suspend(coroutine_handle&lt;T&gt;) const noexcept;
+  };
+
+  template &lt;typename T&gt;
+  struct coroutine_traits {
+      using promise_type = T::promise_type;
+  };
+  }  // namespace std
+
+  struct generator {
+      struct promise_type {
+          std::always_suspend yield_value(int&amp;&amp;);
+          std::always_suspend initial_suspend() const noexcept;
+          std::always_suspend final_suspend() const noexcept;
+          void return_void();
+          void unhandled_exception();
+          generator get_return_object();
+      };
+  };
+
+  generator f() {
+      while (true) {
+          co_yield 10;
+      }
+  }
+
+The matcher coyieldExpr()
+matches co_yield 10
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cudaKernelCallExpr0')"><a name="cudaKernelCallExpr0Anchor">cudaKernelCallExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CUDAKernelCallExpr.html">CUDAKernelCallExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cudaKernelCallExpr0"><pre>Matches CUDA kernel call expression.
 
-Example matches,
-  kernel&lt;&lt;&lt;i,j&gt;&gt;&gt;();
+Given
+  __global__ void kernel() {}
+  void f() {
+    kernel&lt;&lt;&lt;32,32&gt;&gt;&gt;();
+  }
+
+The matcher cudaKernelCallExpr()
+matches kernel&lt;&lt;&lt;i, k&gt;&gt;&gt;()
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxBindTemporaryExpr0')"><a name="cxxBindTemporaryExpr0Anchor">cxxBindTemporaryExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBindTemporaryExpr.html">CXXBindTemporaryExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxBindTemporaryExpr0"><pre>Matches nodes where temporaries are created.
 
-Example matches FunctionTakesString(GetStringByValue())
-    (matcher = cxxBindTemporaryExpr())
-  FunctionTakesString(GetStringByValue());
-  FunctionTakesStringByPointer(GetStringPointer());
+Given
+  struct S {
+    S() { }  // User defined constructor makes S non-POD.
+    ~S() { } // User defined destructor makes it non-trivial.
+  };
+  void test() {
+    const S &amp;s_ref = S(); // Requires a CXXBindTemporaryExpr.
+  }
+
+The matcher cxxBindTemporaryExpr()
+matches the constructor call S().
 </pre></td></tr>
 
 
@@ -1548,49 +1977,71 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="cxxBoolLiteral0"><pre>Matches bool literals.
 
 Example matches true
-  true
+  bool Flag = true;
+
+
+The matcher cxxBoolLiteral() matches true.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxCatchStmt0')"><a name="cxxCatchStmt0Anchor">cxxCatchStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCatchStmt.html">CXXCatchStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxCatchStmt0"><pre>Matches catch statements.
 
+void foo() {
   try {} catch(int i) {}
-cxxCatchStmt()
-  matches 'catch(int i)'
+}
+
+The matcher cxxCatchStmt()
+matches catch(int i) {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxConstCastExpr0')"><a name="cxxConstCastExpr0Anchor">cxxConstCastExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstCastExpr.html">CXXConstCastExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxConstCastExpr0"><pre>Matches a const_cast expression.
 
-Example: Matches const_cast&lt;int*&gt;(&amp;r) in
+Given
   int n = 42;
   const int &amp;r(n);
   int* p = const_cast&lt;int*&gt;(&amp;r);
+
+
+The matcher cxxConstCastExpr()
+matches const_cast&lt;int*&gt;(&amp;r).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxConstructExpr0')"><a name="cxxConstructExpr0Anchor">cxxConstructExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxConstructExpr0"><pre>Matches constructor call expressions (including implicit ones).
 
-Example matches string(ptr, n) and ptr within arguments of f
-    (matcher = cxxConstructExpr())
+Given
+  struct string {
+    string(const char*);
+    string(const char*s, int n);
+  };
   void f(const string &amp;a, const string &amp;b);
-  char *ptr;
-  int n;
-  f(string(ptr, n), ptr);
+  void foo(char *ptr, int n) {
+    f(string(ptr, n), ptr);
+  }
+
+
+The matcher cxxConstructExpr() matches string(ptr, n)
+and ptr within arguments of f .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxDefaultArgExpr0')"><a name="cxxDefaultArgExpr0Anchor">cxxDefaultArgExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXDefaultArgExpr.html">CXXDefaultArgExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxDefaultArgExpr0"><pre>Matches the value of a default argument at the call site.
 
-Example matches the CXXDefaultArgExpr placeholder inserted for the
-    default value of the second parameter in the call expression f(42)
-    (matcher = cxxDefaultArgExpr())
+Given
   void f(int x, int y = 0);
-  f(42);
+  void g() {
+    f(42);
+  }
+
+
+The matcher callExpr(has(cxxDefaultArgExpr()))
+matches the CXXDefaultArgExpr placeholder inserted for the default value
+of the second parameter in the call expression f(42).
 </pre></td></tr>
 
 
@@ -1598,9 +2049,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="cxxDeleteExpr0"><pre>Matches delete expressions.
 
 Given
-  delete X;
-cxxDeleteExpr()
-  matches 'delete X'.
+  void* operator new(decltype(sizeof(void*)));
+  void operator delete(void*);
+  struct X {};
+  void foo() {
+    auto* x = new X;
+    delete x;
+  }
+
+
+The matcher cxxDeleteExpr()
+matches delete x.
 </pre></td></tr>
 
 
@@ -1610,7 +2069,8 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   template &lt;class T&gt; void f() { T t; t.g(); }
-cxxDependentScopeMemberExpr()
+
+The matcher cxxDependentScopeMemberExpr()
   matches t.g
 </pre></td></tr>
 
@@ -1618,53 +2078,83 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxDynamicCastExpr0')"><a name="cxxDynamicCastExpr0Anchor">cxxDynamicCastExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXDynamicCastExpr.html">CXXDynamicCastExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxDynamicCastExpr0"><pre>Matches a dynamic_cast expression.
 
-Example:
-  cxxDynamicCastExpr()
-matches
-  dynamic_cast&lt;D*&gt;(&amp;b);
-in
+Given
   struct B { virtual ~B() {} }; struct D : B {};
   B b;
   D* p = dynamic_cast&lt;D*&gt;(&amp;b);
+
+
+The matcher cxxDynamicCastExpr()
+matches dynamic_cast&lt;D*&gt;(&amp;b).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxFoldExpr0')"><a name="cxxFoldExpr0Anchor">cxxFoldExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxFoldExpr0"><pre>Matches C++17 fold expressions.
 
-Example matches `(0 + ... + args)`:
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+
+The matcher cxxFoldExpr() matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxForRangeStmt0')"><a name="cxxForRangeStmt0Anchor">cxxForRangeStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXForRangeStmt.html">CXXForRangeStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxForRangeStmt0"><pre>Matches range-based for statements.
 
-cxxForRangeStmt() matches 'for (auto a : i)'
-  int i[] =  {1, 2, 3}; for (auto a : i);
-  for(int j = 0; j &lt; 5; ++j);
+Given
+  void foo() {
+    int i[] =  {1, 2, 3}; for (auto a : i);
+    for(int j = 0; j &lt; 5; ++j);
+  }
+
+The matcher cxxForRangeStmt()
+matches for (auto a : i);
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxFunctionalCastExpr0')"><a name="cxxFunctionalCastExpr0Anchor">cxxFunctionalCastExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxFunctionalCastExpr0"><pre>Matches functional cast expressions
 
-Example: Matches Foo(bar);
-  Foo f = bar;
-  Foo g = (Foo) bar;
-  Foo h = Foo(bar);
+Given
+  struct Foo {
+    Foo(int x);
+  };
+
+  void foo(int bar) {
+    Foo f = bar;
+    Foo g = (Foo) bar;
+    Foo h = Foo(bar);
+  }
+
+
+The matcher cxxFunctionalCastExpr()
+matches Foo(bar).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxMemberCallExpr0')"><a name="cxxMemberCallExpr0Anchor">cxxMemberCallExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMemberCallExpr.html">CXXMemberCallExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxMemberCallExpr0"><pre>Matches member call expressions.
 
-Example matches x.y()
-  X x;
-  x.y();
+Given
+  struct X {
+    void y();
+    void m() { y(); }
+  };
+  void f();
+  void g() {
+    X x;
+    x.y();
+    f();
+  }
+
+
+The matcher cxxMemberCallExpr() matches x.y() and
+y(), but not f().
 </pre></td></tr>
 
 
@@ -1672,9 +2162,15 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="cxxNewExpr0"><pre>Matches new expressions.
 
 Given
-  new X;
-cxxNewExpr()
-  matches 'new X'.
+  void* operator new(decltype(sizeof(void*)));
+  struct X {};
+  void foo() {
+    auto* x = new X;
+  }
+
+
+The matcher cxxNewExpr()
+matches new X.
 </pre></td></tr>
 
 
@@ -1687,14 +2183,24 @@ <h2 id="decl-matchers">Node Matchers</h2>
   bool c() noexcept(false);
   bool d() noexcept(noexcept(a()));
   bool e = noexcept(b()) || noexcept(c());
-cxxNoexceptExpr()
-  matches `noexcept(a())`, `noexcept(b())` and `noexcept(c())`.
-  doesn't match the noexcept specifier in the declarations a, b, c or d.
+
+The matcher cxxNoexceptExpr()
+matches noexcept(a()), noexcept(b()) and
+noexcept(c()), but does not match the noexcept specifier in the
+declarations a, b, c or d.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxNullPtrLiteralExpr0')"><a name="cxxNullPtrLiteralExpr0Anchor">cxxNullPtrLiteralExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNullPtrLiteralExpr.html">CXXNullPtrLiteralExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxNullPtrLiteralExpr0"><pre>Matches nullptr literal.
+
+Given
+  int a = 0;
+  int* b = 0;
+  int *c = nullptr;
+
+
+The matcher cxxNullPtrLiteralExpr() matches nullptr.
 </pre></td></tr>
 
 
@@ -1706,11 +2212,16 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Currently it does not match operators such as new delete.
 FIXME: figure out why these do not match?
 
-Example matches both operator&lt;&lt;((o &lt;&lt; b), c) and operator&lt;&lt;(o, b)
-    (matcher = cxxOperatorCallExpr())
+Given
+  struct ostream;
   ostream &amp;operator&lt;&lt; (ostream &amp;out, int i) { };
-  ostream &amp;o; int b = 1, c = 1;
-  o &lt;&lt; b &lt;&lt; c;
+  void f(ostream&amp; o, int b, int c) {
+    o &lt;&lt; b &lt;&lt; c;
+  }
+
+
+The matcher cxxOperatorCallExpr() matches o &lt;&lt; b &lt;&lt; c
+and o &lt;&lt; b.
 See also the binaryOperation() matcher for more-general matching of binary
 uses of this AST node.
 </pre></td></tr>
@@ -1725,6 +2236,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Example matches reinterpret_cast&lt;char*&gt;(&amp;p) in
   void* p = reinterpret_cast&lt;char*&gt;(&amp;p);
+
+
+The matcher cxxReinterpretCastExpr()
+matches reinterpret_cast&lt;char*&gt;(&amp;p).
 </pre></td></tr>
 
 
@@ -1732,16 +2247,20 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="cxxRewrittenBinaryOperator0"><pre>Matches rewritten binary operators
 
 Example matches use of "&lt;":
-  #include &lt;compare&gt;
   struct HasSpaceshipMem {
     int a;
-    constexpr auto operator&lt;=&gt;(const HasSpaceshipMem&amp;) const = default;
+    constexpr bool operator==(const HasSpaceshipMem&amp;) const = default;
   };
   void compare() {
     HasSpaceshipMem hs1, hs2;
-    if (hs1 &lt; hs2)
+    if (hs1 != hs2)
         return;
   }
+
+
+The matcher cxxRewrittenBinaryOperator() matches
+hs1 != hs2.
+
 See also the binaryOperation() matcher for more-general matching
 of this AST node.
 </pre></td></tr>
@@ -1753,12 +2272,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 See also: hasDestinationType
 See also: reinterpretCast
 
-Example:
-  cxxStaticCastExpr()
-matches
-  static_cast&lt;long&gt;(8)
-in
+Given
   long eight(static_cast&lt;long&gt;(8));
+
+
+The matcher cxxStaticCastExpr()
+matches static_cast&lt;long&gt;(8).
 </pre></td></tr>
 
 
@@ -1766,69 +2285,110 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="cxxStdInitializerListExpr0"><pre>Matches C++ initializer list expressions.
 
 Given
-  std::vector&lt;int&gt; a({ 1, 2, 3 });
-  std::vector&lt;int&gt; b = { 4, 5 };
+  namespace std {
+    template &lt;typename T&gt;
+    class initializer_list {
+      const T* begin;
+      const T* end;
+    };
+  }
+  template &lt;typename T&gt; class vector {
+    public: vector(std::initializer_list&lt;T&gt;) {}
+  };
+
+  vector&lt;int&gt; a({ 1, 2, 3 });
+  vector&lt;int&gt; b = { 4, 5 };
   int c[] = { 6, 7 };
-  std::pair&lt;int, int&gt; d = { 8, 9 };
-cxxStdInitializerListExpr()
-  matches "{ 1, 2, 3 }" and "{ 4, 5 }"
+  struct pair { int x; int y; };
+  pair d = { 8, 9 };
+
+The matcher cxxStdInitializerListExpr()
+matches { 1, 2, 3 } and { 4, 5 }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxTemporaryObjectExpr0')"><a name="cxxTemporaryObjectExpr0Anchor">cxxTemporaryObjectExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxTemporaryObjectExpr0"><pre>Matches functional cast expressions having N != 1 arguments
 
-Example: Matches Foo(bar, bar)
-  Foo h = Foo(bar, bar);
+Given
+  struct Foo {
+    Foo(int x, int y);
+  };
+
+  void foo(int bar) {
+    Foo h = Foo(bar, bar);
+  }
+
+
+The matcher cxxTemporaryObjectExpr()
+matches Foo(bar, bar).
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxThisExpr0')"><a name="cxxThisExpr0Anchor">cxxThisExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXThisExpr.html">CXXThisExpr</a>&gt;...</td></tr>
-<tr><td colspan="4" class="doc" id="cxxThisExpr0"><pre>Matches implicit and explicit this expressions.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxThisExpr0')"><a name="cxxThisExpr0Anchor">cxxThisExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXThisExpr.html">CXXThisExpr</a>&gt;...</td></tr>
+<tr><td colspan="4" class="doc" id="cxxThisExpr0"><pre>Matches implicit and explicit this expressions.
+
+Given
+  struct foo {
+    int i;
+    int f() { return i; }
+    int g() { return this-&gt;i; }
+  };
+
 
-Example matches the implicit this expression in "return i".
-    (matcher = cxxThisExpr())
-struct foo {
-  int i;
-  int f() { return i; }
-};
+The matcher cxxThisExpr()
+matches this of this-&gt;i and the implicit this expression
+of i.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxThrowExpr0')"><a name="cxxThrowExpr0Anchor">cxxThrowExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXThrowExpr.html">CXXThrowExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxThrowExpr0"><pre>Matches throw expressions.
 
+void foo() {
   try { throw 5; } catch(int i) {}
-cxxThrowExpr()
-  matches 'throw 5'
+}
+
+The matcher cxxThrowExpr()
+matches throw 5
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxTryStmt0')"><a name="cxxTryStmt0Anchor">cxxTryStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTryStmt.html">CXXTryStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxTryStmt0"><pre>Matches try statements.
 
+void foo() {
   try {} catch(int i) {}
-cxxTryStmt()
-  matches 'try {}'
+}
+
+The matcher cxxTryStmt()
+matches try {} catch(int i) {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('cxxUnresolvedConstructExpr0')"><a name="cxxUnresolvedConstructExpr0Anchor">cxxUnresolvedConstructExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="cxxUnresolvedConstructExpr0"><pre>Matches unresolved constructor call expressions.
 
-Example matches T(t) in return statement of f
-    (matcher = cxxUnresolvedConstructExpr())
+Given
   template &lt;typename T&gt;
   void f(const T&amp; t) { return T(t); }
+
+
+The matcher cxxUnresolvedConstructExpr() matches
+T(t).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('declRefExpr0')"><a name="declRefExpr0Anchor">declRefExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="declRefExpr0"><pre>Matches expressions that refer to declarations.
 
-Example matches x in if (x)
-  bool x;
-  if (x) {}
+Given
+  void f(bool x) {
+    if (x) {}
+  }
+
+
+The matcher declRefExpr() matches x.
 </pre></td></tr>
 
 
@@ -1836,9 +2396,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="declStmt0"><pre>Matches declaration statements.
 
 Given
-  int a;
-declStmt()
-  matches 'int a'.
+  void foo() {
+    int a;
+  }
+The matcher declStmt()
+matches int a;.
 </pre></td></tr>
 
 
@@ -1846,22 +2408,75 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="defaultStmt0"><pre>Matches default statements inside switch statements.
 
 Given
+void foo(int a) {
   switch(a) { case 42: break; default: break; }
-defaultStmt()
-  matches 'default:'.
+}
+The matcher defaultStmt()
+matches default: break.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('dependentCoawaitExpr0')"><a name="dependentCoawaitExpr0Anchor">dependentCoawaitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DependentCoawaitExpr.html">DependentCoawaitExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="dependentCoawaitExpr0"><pre>Matches co_await expressions where the type of the promise is dependent
+
+Given
+  namespace std {
+  template &lt;typename T = void&gt;
+  struct coroutine_handle {
+      static constexpr coroutine_handle from_address(void* addr) {
+        return {};
+      }
+  };
+
+  struct always_suspend {
+      bool await_ready() const noexcept;
+      bool await_resume() const noexcept;
+      template &lt;typename T&gt;
+      bool await_suspend(coroutine_handle&lt;T&gt;) const noexcept;
+  };
+
+  template &lt;typename T&gt;
+  struct coroutine_traits {
+      using promise_type = T::promise_type;
+  };
+  }  // namespace std
+
+  template &lt;typename T&gt;
+  struct generator {
+      struct promise_type {
+          std::always_suspend yield_value(int&amp;&amp;);
+          std::always_suspend initial_suspend() const noexcept;
+          std::always_suspend final_suspend() const noexcept;
+          void return_void();
+          void unhandled_exception();
+          generator get_return_object();
+      };
+  };
+
+  template &lt;typename T&gt;
+  std::always_suspend h();
+
+  template &lt;&gt;
+  std::always_suspend h&lt;void&gt;();
+
+  template&lt;typename T&gt;
+  generator&lt;T&gt; g() { co_await h&lt;T&gt;(); }
+
+The matcher dependentCoawaitExpr()
+matches co_await h&lt;T&gt;().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('designatedInitExpr0')"><a name="designatedInitExpr0Anchor">designatedInitExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DesignatedInitExpr.html">DesignatedInitExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="designatedInitExpr0"><pre>Matches C99 designated initializer expressions [C99 6.7.8].
 
-Example: Matches { [2].y = 1.0, [0].x = 1.0 }
-  point ptarray[10] = { [2].y = 1.0, [0].x = 1.0 };
+Example: Given
+  struct point2 { double x; double y; };
+  struct point2 ptarray[10] = { [0].x = 1.0 };
+  struct point2 pt = { .x = 2.0 };
+
+The matcher designatedInitExpr()
+matches [0].x = 1.0 and .x = 2.0.
 </pre></td></tr>
 
 
@@ -1869,9 +2484,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="doStmt0"><pre>Matches do statements.
 
 Given
+void foo() {
   do {} while (true);
-doStmt()
-  matches 'do {} while(true)'
+}
+
+The matcher doStmt()
+matches do {} while (true)
 </pre></td></tr>
 
 
@@ -1889,18 +2507,36 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 See also: hasDestinationType.
 
-Example: matches all five of the casts in
-  int((int)(reinterpret_cast&lt;int&gt;(static_cast&lt;int&gt;(const_cast&lt;int&gt;(42)))))
-but does not match the implicit conversion in
-  long ell = 42;
+  struct S {};
+  const S* s;
+  S* s2 = const_cast&lt;S*&gt;(s);
+
+  const int val = 0;
+  char val0 = val;
+  char val1 = (char)val;
+  char val2 = static_cast&lt;char&gt;(val);
+  int* val3 = reinterpret_cast&lt;int*&gt;(val);
+  char val4 = char(val);
+
+
+The matcher explicitCastExpr()
+matches (char)val, static_cast&lt;char&gt;(val),
+reinterpret_cast&lt;int*&gt;(val), const_cast&lt;S*&gt;(s)
+and char(val), but not the initialization of val0 with
+val.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('expr0')"><a name="expr0Anchor">expr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="expr0"><pre>Matches expressions.
 
-Example matches x()
-  void f() { x(); }
+Given
+  int f(int x, int y) { return x + y; }
+
+The matcher expr() matches x + y once,
+x twice and y twice, matching the
+DeclRefExpr , and the ImplicitCastExpr that does an l- to r-value
+cast.
 </pre></td></tr>
 
 
@@ -1909,12 +2545,33 @@ <h2 id="decl-matchers">Node Matchers</h2>
 of the sub-expression's evaluation.
 
 Example matches std::string()
-  const std::string str = std::string();
+  struct A { ~A(); };
+  void f(A);
+  void g(A&amp;);
+  void h() {
+    A a = A{};
+    f(A{});
+    f(a);
+    g(a);
+  }
+
+
+The matcher exprWithCleanups() matches A{},
+f(A{}) and f(a),
+but does not match passing g(a).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('fixedPointLiteral0')"><a name="fixedPointLiteral0Anchor">fixedPointLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FixedPointLiteral.html">FixedPointLiteral</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="fixedPointLiteral0"><pre>Matches fixed point literals
+
+Given
+  void f() {
+    0.0k;
+  }
+
+
+The matcher fixedPointLiteral() matches 0.0k.
 </pre></td></tr>
 
 
@@ -1922,27 +2579,62 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="floatLiteral0"><pre>Matches float literals of all sizes / encodings, e.g.
 1.0, 1.0f, 1.0L and 1e10.
 
-Does not match implicit conversions such as
-  float a = 10;
+Given
+  int a = 1.0;
+  int b = 1.0F;
+  int c = 1.0L;
+  int d = 1e10;
+  int e = 1;
+
+The matcher floatLiteral() matches
+1.0, 1.0F, 1.0L and 1e10, but does not match
+1.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('forStmt0')"><a name="forStmt0Anchor">forStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ForStmt.html">ForStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="forStmt0"><pre>Matches for statements.
 
-Example matches 'for (;;) {}'
-  for (;;) {}
-  int i[] =  {1, 2, 3}; for (auto a : i);
+Given
+  void foo() {
+    for (;;) {}
+    int i[] =  {1, 2, 3}; for (auto a : i);
+  }
+
+
+The matcher forStmt() matches for (;;) {},
+but not for (auto a : i);.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('genericSelectionExpr0')"><a name="genericSelectionExpr0Anchor">genericSelectionExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1GenericSelectionExpr.html">GenericSelectionExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="genericSelectionExpr0"><pre>Matches C11 _Generic expression.
+
+Given
+  double fdouble(double);
+  float ffloat(float);
+  #define GENERIC_MACRO(X) _Generic((X), double: fdouble, float: ffloat)(X)
+
+  void f() {
+      GENERIC_MACRO(0.0);
+      GENERIC_MACRO(0.0F);
+  }
+
+
+The matcher genericSelectionExpr() matches
+the generic selection expression that is expanded in
+GENERIC_MACRO(0.0) and GENERIC_MACRO(0.0F).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('gnuNullExpr0')"><a name="gnuNullExpr0Anchor">gnuNullExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1GNUNullExpr.html">GNUNullExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="gnuNullExpr0"><pre>Matches GNU __null expression.
+
+Given
+  auto val = __null;
+
+
+The matcher gnuNullExpr() matches __null.
 </pre></td></tr>
 
 
@@ -1950,24 +2642,39 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="gotoStmt0"><pre>Matches goto statements.
 
 Given
+void bar();
+void foo() {
   goto FOO;
   FOO: bar();
-gotoStmt()
-  matches 'goto FOO'
+}
+The matcher gotoStmt()
+matches goto FOO
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('ifStmt0')"><a name="ifStmt0Anchor">ifStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IfStmt.html">IfStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="ifStmt0"><pre>Matches if statements.
 
-Example matches 'if (x) {}'
-  if (x) {}
+Given
+  void foo(int x) {
+    if (x) {}
+  }
+
+The matcher ifStmt() matches if (x) {}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('imaginaryLiteral0')"><a name="imaginaryLiteral0Anchor">imaginaryLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ImaginaryLiteral.html">ImaginaryLiteral</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="imaginaryLiteral0"><pre>Matches imaginary literals, which are based on integer and floating
 point literals e.g.: 1i, 1.0i
+
+Given
+  auto a = 1i;
+  auto b = 1.0i;
+
+
+The matcher imaginaryLiteral() matches 1i and
+1.0i.
 </pre></td></tr>
 
 
@@ -1976,6 +2683,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 This matches many different places, including function call return value
 eliding, as well as any type conversions.
+
+void f(int);
+void g(int val1, int val2) {
+  unsigned int a = val1;
+  f(val2);
+}
+
+The matcher implicitCastExpr()
+matches val1 for the implicit cast from an l- to an r-value
+and for the cast to int}, f for the function pointer
+decay, and val2 for the cast from an l- to an r-value.
 </pre></td></tr>
 
 
@@ -1983,9 +2701,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="implicitValueInitExpr0"><pre>Matches implicit initializers of init list expressions.
 
 Given
-  point ptarray[10] = { [2].y = 1.0, [2].x = 2.0, [0].x = 1.0 };
-implicitValueInitExpr()
-  matches "[0].y" (implicitly)
+  struct point { double x; double y; };
+  struct point pt = { .x = 42.0 };
+The matcher
+initListExpr(has(implicitValueInitExpr().bind("implicit")))
+matches { .x = 42.0 }.
 </pre></td></tr>
 
 
@@ -1995,9 +2715,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   int a[] = { 1, 2 };
   struct B { int x, y; };
-  B b = { 5, 6 };
-initListExpr()
-  matches "{ 1, 2 }" and "{ 5, 6 }"
+  struct B b = { 5, 6 };
+The matcher initListExpr()
+matches { 1, 2 } and { 5, 6 }
 </pre></td></tr>
 
 
@@ -2006,6 +2726,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 1, 1L, 0x1 and 1U.
 
 Does not match character-encoded integers such as L'a'.
+
+Given
+  int a = 1;
+  int b = 1L;
+  int c = 0x1;
+  int d = 1U;
+  int e = 1.0;
+
+The matcher integerLiteral() matches
+1, 1L, 0x1 and 1U, but does not match
+1.0.
 </pre></td></tr>
 
 
@@ -2013,18 +2744,26 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="labelStmt0"><pre>Matches label statements.
 
 Given
+void bar();
+void foo() {
   goto FOO;
   FOO: bar();
-labelStmt()
-  matches 'FOO:'
+}
+The matcher labelStmt()
+matches FOO: bar()
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('lambdaExpr0')"><a name="lambdaExpr0Anchor">lambdaExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1LambdaExpr.html">LambdaExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="lambdaExpr0"><pre>Matches lambda expressions.
 
-Example matches [&amp;](){return 5;}
-  [&amp;](){return 5;}
+Given
+  void f() {
+    []() { return 5; };
+  }
+
+
+The matcher lambdaExpr() matches []() { return 5; }.
 </pre></td></tr>
 
 
@@ -2035,12 +2774,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
   struct T {void func();};
   T f();
   void g(T);
-materializeTemporaryExpr() matches 'f()' in these statements
-  T u(f());
-  g(f());
-  f().func();
-but does not match
-  f();
+  void foo() {
+    T u(f());
+    g(f());
+    f().func();
+    f(); // does not match
+  }
+
+The matcher materializeTemporaryExpr() matches
+f() three times before C++17 and it
+matches f() time with C++17 and later, but
+it does not match the f() in the last line in any version.
 </pre></td></tr>
 
 
@@ -2052,17 +2796,20 @@ <h2 id="decl-matchers">Node Matchers</h2>
     void x() { this-&gt;x(); x(); Y y; y.x(); a; this-&gt;b; Y::b; }
     int a; static int b;
   };
-memberExpr()
-  matches this-&gt;x, x, y.x, a, this-&gt;b
+
+The matcher memberExpr()
+matches this-&gt;x, x, y.x, a, this-&gt;b.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('nullStmt0')"><a name="nullStmt0Anchor">nullStmt</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NullStmt.html">NullStmt</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="nullStmt0"><pre>Matches null statements.
 
+void foo() {
   foo();;
-nullStmt()
-  matches the second ';'
+}
+The matcher nullStmt()
+matches the second ;
 </pre></td></tr>
 
 
@@ -2072,6 +2819,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches @catch
   @try {}
   @catch (...) {}
+
 </pre></td></tr>
 
 
@@ -2081,19 +2829,23 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches @finally
   @try {}
   @finally {}
+
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('objcIvarRefExpr0')"><a name="objcIvarRefExpr0Anchor">objcIvarRefExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCIvarRefExpr.html">ObjCIvarRefExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="objcIvarRefExpr0"><pre>Matches a reference to an ObjCIvar.
 
-Example: matches "a" in "init" method:
+Given
 @implementation A {
   NSString *a;
 }
 - (void) init {
   a = @"hello";
 }
+
+
+The matcher objcIvarRefExpr() matches a.
 </pre></td></tr>
 
 
@@ -2105,6 +2857,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 "initWithString" instance method on the object returned from
 NSString's "alloc". This matcher should match both message sends.
   [[NSString alloc] initWithString:@"Hello"]
+
+
+The matcher objcMessageExpr() matches
+[[NSString alloc] initWithString:@"Hello"]
 </pre></td></tr>
 
 
@@ -2113,6 +2869,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Example matches @"abcd"
   NSString *s = @"abcd";
+
 </pre></td></tr>
 
 
@@ -2120,6 +2877,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="objcThrowStmt0"><pre>Matches Objective-C statements.
 
 Example matches @throw obj;
+
 </pre></td></tr>
 
 
@@ -2129,6 +2887,7 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Example matches @try
   @try {}
   @catch (...) {}
+
 </pre></td></tr>
 
 
@@ -2136,13 +2895,19 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="ompExecutableDirective0"><pre>Matches any ``#pragma omp`` executable directive.
 
 Given
+  void foo() {
+    #pragma omp parallel
+      {}
+    #pragma omp parallel default(none)
+      {
+        #pragma omp taskyield
+      }
+  }
 
-  #pragma omp parallel
-  #pragma omp parallel default(none)
-  #pragma omp taskyield
-
-``ompExecutableDirective()`` matches ``omp parallel``,
-``omp parallel default(none)`` and ``omp taskyield``.
+The matcher ompExecutableDirective()
+matches #pragma omp parallel,
+#pragma omp parallel default(none)
+and #pragma omp taskyield.
 </pre></td></tr>
 
 
@@ -2151,17 +2916,27 @@ <h2 id="decl-matchers">Node Matchers</h2>
 to reference another expressions and can be met
 in BinaryConditionalOperators, for example.
 
-Example matches 'a'
-  (a ?: c) + 42;
+Given
+  int f(int a, int b) {
+    return (a ?: b) + 42;
+  }
+
+
+The matcher opaqueValueExpr() matches a twice,
+once for the check and once for the expression of the true path.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('parenExpr0')"><a name="parenExpr0Anchor">parenExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ParenExpr.html">ParenExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="parenExpr0"><pre>Matches parentheses used in expressions.
 
-Example matches (foo() + 1)
+Given
   int foo() { return 1; }
-  int a = (foo() + 1);
+  int bar() {
+    int a = (foo() + 1);
+  }
+
+The matcher parenExpr() matches (foo() + 1).
 </pre></td></tr>
 
 
@@ -2177,8 +2952,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
       int a = 0, b = 1; int i = (a, b);
     }
   };
-parenListExpr() matches "*this" but NOT matches (a, b) because (a, b)
-has a predefined type and is a ParenExpr, not a ParenListExpr.
+
+The matcher parenListExpr()
+matches (*this),
+but does not match (a, b)
+because (a, b) has a predefined type and is a ParenExpr, not a
+ParenListExpr.
 </pre></td></tr>
 
 
@@ -2186,7 +2965,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="predefinedExpr0"><pre>Matches predefined identifier expressions [C99 6.4.2.2].
 
 Example: Matches __func__
-  printf("%s", __func__);
+  void f() {
+    const char* func_name = __func__;
+  }
+
+The matcher predefinedExpr()
+matches __func__.
 </pre></td></tr>
 
 
@@ -2194,9 +2978,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="returnStmt0"><pre>Matches return statements.
 
 Given
+int foo() {
   return 1;
-returnStmt()
-  matches 'return 1'
+}
+The matcher returnStmt()
+matches return 1
 </pre></td></tr>
 
 
@@ -2204,26 +2990,35 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="stmt0"><pre>Matches statements.
 
 Given
-  { ++a; }
-stmt()
-  matches both the compound statement '{ ++a; }' and '++a'.
+  void foo(int a) { { ++a; } }
+The matcher stmt()
+matches the function body itself { { ++a; } }, the compound
+statement { ++a; }, the expression ++a and a.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('stmtExpr0')"><a name="stmtExpr0Anchor">stmtExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1StmtExpr.html">StmtExpr</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="stmtExpr0"><pre>Matches statement expression (GNU extension).
 
-Example match: ({ int X = 4; X; })
-  int C = ({ int X = 4; X; });
+Given
+  void f() {
+    int C = ({ int X = 4; X; });
+  }
+
+The matcher stmtExpr() matches ({ int X = 4; X; }).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('stringLiteral0')"><a name="stringLiteral0Anchor">stringLiteral</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1StringLiteral.html">StringLiteral</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="stringLiteral0"><pre>Matches string literals (also matches wide string literals).
 
-Example matches "abcd", L"abcd"
+Given
   char *s = "abcd";
   wchar_t *ws = L"abcd";
+
+
+The matcher stringLiteral() matches "abcd" and
+L"abcd".
 </pre></td></tr>
 
 
@@ -2234,8 +3029,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   template &lt;int N&gt;
   struct A { static const int n = N; };
   struct B : public A&lt;42&gt; {};
-substNonTypeTemplateParmExpr()
-  matches "N" in the right-hand side of "static const int n = N;"
+
+The matcher substNonTypeTemplateParmExpr()
+matches N in the right-hand side of "static const int n = N;"
 </pre></td></tr>
 
 
@@ -2243,9 +3039,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="switchCase0"><pre>Matches case and default statements inside switch statements.
 
 Given
+void foo(int a) {
   switch(a) { case 42: break; default: break; }
-switchCase()
-  matches 'case 42:' and 'default:'.
+}
+The matcher switchCase()
+matches case 42: break and default: break
 </pre></td></tr>
 
 
@@ -2253,9 +3051,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="switchStmt0"><pre>Matches switch statements.
 
 Given
+void foo(int a) {
   switch(a) { case 42: break; default: break; }
-switchStmt()
-  matches 'switch(a)'.
+}
+The matcher switchStmt()
+matches switch(a) { case 42: break; default: break; }.
 </pre></td></tr>
 
 
@@ -2263,10 +3063,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="unaryExprOrTypeTraitExpr0"><pre>Matches sizeof (C99), alignof (C++11) and vec_step (OpenCL)
 
 Given
-  Foo x = bar;
+  int x = 42;
   int y = sizeof(x) + alignof(x);
-unaryExprOrTypeTraitExpr()
-  matches sizeof(x) and alignof(x)
+
+The matcher unaryExprOrTypeTraitExpr()
+matches sizeof(x) and alignof(x)
 </pre></td></tr>
 
 
@@ -2274,7 +3075,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="unaryOperator0"><pre>Matches unary operator expressions.
 
 Example matches !a
-  !a || b
+  void foo(bool a, bool b) {
+    !a || b;
+  }
+
+
+The matcher unaryOperator() matches !a.
 </pre></td></tr>
 
 
@@ -2289,8 +3095,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   void bar() {
     foo&lt;T&gt;();
   }
-unresolvedLookupExpr()
-  matches foo&lt;T&gt;() </pre></td></tr>
+
+The matcher unresolvedLookupExpr()
+matches foo&lt;T&gt;.
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('unresolvedMemberExpr0')"><a name="unresolvedMemberExpr0Anchor">unresolvedMemberExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnresolvedMemberExpr.html">UnresolvedMemberExpr</a>&gt;...</td></tr>
@@ -2302,8 +3110,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
     void g();
   };
   template &lt;class T&gt; void h() { X x; x.f&lt;T&gt;(); x.g(); }
-unresolvedMemberExpr()
-  matches x.f&lt;T&gt;
+
+The matcher unresolvedMemberExpr()
+matches x.f&lt;T&gt;
 </pre></td></tr>
 
 
@@ -2311,6 +3120,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="userDefinedLiteral0"><pre>Matches user defined literal operator call.
 
 Example match: "foo"_suffix
+Given
+  float operator ""_foo(long double);
+  float a = 1234.5_foo;
+
+
+The matcher userDefinedLiteral() matches 1234.5_foo.
 </pre></td></tr>
 
 
@@ -2318,9 +3133,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="whileStmt0"><pre>Matches while statements.
 
 Given
+void foo() {
   while (true) {}
-whileStmt()
-  matches 'while (true) {}'.
+}
+
+The matcher whileStmt()
+matches while (true) {}.
 </pre></td></tr>
 
 
@@ -2330,8 +3148,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   template &lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
-templateArgumentLoc()
-  matches 'int' in C&lt;int&gt;.
+
+The matcher templateArgumentLoc()
+matches int in C&lt;int&gt;.
 </pre></td></tr>
 
 
@@ -2341,8 +3160,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   template &lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
-templateArgument()
-  matches 'int' in C&lt;int&gt;.
+
+The matcher
+templateSpecializationType(hasAnyTemplateArgument(templateArgument()))
+matches C&lt;int&gt;.
 </pre></td></tr>
 
 
@@ -2350,10 +3171,14 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="templateName0"><pre>Matches template name.
 
 Given
-  template &lt;typename T&gt; class X { };
-  X&lt;int&gt; xi;
-templateName()
-  matches 'X' in X&lt;int&gt;.
+  template&lt;template &lt;typename&gt; class S&gt; class X {};
+  template&lt;typename T&gt; class Y {};
+  X&lt;Y&gt; xi;
+
+The matcher
+classTemplateSpecializationDecl(hasAnyTemplateArgument(
+              refersToTemplate(templateName())))
+matches the specialization class X&lt;Y&gt;
 </pre></td></tr>
 
 
@@ -2363,8 +3188,8 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   struct s {};
   struct s ss;
-elaboratedTypeLoc()
-  matches the `TypeLoc` of the variable declaration of `ss`.
+The matcher elaboratedTypeLoc()
+matches the type struct s of ss.
 </pre></td></tr>
 
 
@@ -2373,8 +3198,8 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   int* x;
-pointerTypeLoc()
-  matches `int*`.
+The matcher pointerTypeLoc()
+  matches int*.
 </pre></td></tr>
 
 
@@ -2383,8 +3208,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   const int x = 0;
-qualifiedTypeLoc()
-  matches `const int`.
+
+The matcher qualifiedTypeLoc()
+matches the type of the variable declaration x . However, the
+current implementation of QualifiedTypeLoc does not store the source
+locations for the qualifiers of the type int.
 </pre></td></tr>
 
 
@@ -2395,8 +3223,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   int x = 3;
   int&amp; l = x;
   int&amp;&amp; r = 3;
-referenceTypeLoc()
-  matches `int&amp;` and `int&amp;&amp;`.
+
+
+The matcher referenceTypeLoc()
+  matches int&amp; and int&amp;&amp;.
 </pre></td></tr>
 
 
@@ -2406,13 +3236,25 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   template &lt;typename T&gt; class C {};
   C&lt;char&gt; var;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(typeLoc())))
-  matches `C&lt;char&gt; var`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(typeLoc())))))
+matches C&lt;char&gt; var.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('typeLoc0')"><a name="typeLoc0Anchor">typeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="typeLoc0"><pre>Matches TypeLocs in the clang AST.
+
+That is, information about a type and where it was written.
+
+  void foo(int val);
+
+The matcher declaratorDecl(hasTypeLoc(typeLoc().bind("type")))
+matches void foo(int val) and int val, with
+typeLoc() matching void and
+int respectively.
 </pre></td></tr>
 
 
@@ -2423,8 +3265,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   int a[] = { 2, 3 };
   int b[4];
   void f() { int c[a[0]]; }
-arrayType()
-  matches "int a[]", "int b[4]" and "int c[a[0]]";
+The matcher arrayType()
+int[4], int[a[0]] and
+int[];
 </pre></td></tr>
 
 
@@ -2433,20 +3276,25 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   _Atomic(int) i;
-atomicType()
-  matches "_Atomic(int) i"
+The matcher atomicType()
+_Atomic(int)
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('autoType0')"><a name="autoType0Anchor">autoType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AutoType.html">AutoType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="autoType0"><pre>Matches types nodes representing C++11 auto types.
 
-Given:
-  auto n = 4;
-  int v[] = { 2, 3 }
-  for (auto i : v) { }
-autoType()
-  matches "auto n" and "auto i"
+Given
+  void foo() {
+    auto n = 4;
+    int v[] = { 2, 3 };
+    for (auto i : v) { };
+  }
+
+The matcher autoType()
+matches the auto of n and i ,
+as well as the auto types for the implicitly generated code of the range-for
+loop (for the range, the begin iterator and the end iterator).
 </pre></td></tr>
 
 
@@ -2462,13 +3310,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="builtinType0"><pre>Matches builtin Types.
 
 Given
-  struct A {};
-  A a;
+  enum E { Ok };
+  enum E e;
   int b;
   float c;
-  bool d;
-builtinType()
-  matches "int b", "float c" and "bool d"
+The matcher varDecl(hasType(builtinType()))
+matches int b and float c.
 </pre></td></tr>
 
 
@@ -2477,8 +3324,8 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   _Complex float f;
-complexType()
-  matches "_Complex float f"
+The matcher complexType()
+_Complex float
 </pre></td></tr>
 
 
@@ -2486,37 +3333,38 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="constantArrayType0"><pre>Matches C arrays with a specified constant size.
 
 Given
-  void() {
+  void foo() {
     int a[2];
     int b[] = { 2, 3 };
     int c[b[0]];
   }
-constantArrayType()
-  matches "int a[2]"
+The matcher constantArrayType()
+int[2]
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('decayedType0')"><a name="decayedType0Anchor">decayedType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecayedType.html">DecayedType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="decayedType0"><pre>Matches decayed type
-Example matches i[] in declaration of f.
-    (matcher = valueDecl(hasType(decayedType(hasDecayedType(pointerType())))))
-Example matches i[1].
-    (matcher = expr(hasType(decayedType(hasDecayedType(pointerType())))))
   void f(int i[]) {
     i[1] = 0;
   }
-</pre></td></tr>
+The matcher
+valueDecl(hasType(decayedType(hasDecayedType(pointerType()))))
+matches int i[] in declaration of The matcher
+expr(hasType(decayedType(hasDecayedType(pointerType()))))
+matches i in </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('decltypeType0')"><a name="decltypeType0Anchor">decltypeType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecltypeType.html">DecltypeType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="decltypeType0"><pre>Matches types nodes representing C++11 decltype(&lt;expr&gt;) types.
 
-Given:
+Given
   short i = 1;
   int j = 42;
   decltype(i + j) result = i + j;
-decltypeType()
-  matches "decltype(i + j)"
+
+The matcher decltypeType()
+decltype(i + j)
 </pre></td></tr>
 
 
@@ -2529,8 +3377,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   class C { public: C(T); };
 
   C c(123);
-deducedTemplateSpecializationType() matches the type in the declaration
-of the variable c.
+
+The matcher deducedTemplateSpecializationType() matches the type
+C of the declaration of the variable c.
 </pre></td></tr>
 
 
@@ -2542,8 +3391,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   class array {
     T data[Size];
   };
-dependentSizedArrayType()
-  matches "T data[Size]"
+
+The matcher dependentSizedArrayType()
+T[Size]
 </pre></td></tr>
 
 
@@ -2556,8 +3406,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   class vector {
     typedef T __attribute__((ext_vector_type(Size))) type;
   };
-dependentSizedExtVectorType()
-  matches "T __attribute__((ext_vector_type(Size)))"
+
+The matcher dependentSizedExtVectorType()
+T __attribute__((ext_vector_type(Size)))
 </pre></td></tr>
 
 
@@ -2573,11 +3424,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
   }
   class C {};
 
-  class C c;
+  C c;
   N::M::D d;
 
-elaboratedType() matches the type of the variable declarations of both
-c and d.
+
+The matcher elaboratedType() matches the type
+C three times. Once for the type of the
+variable c, once for the type of the class definition and once for the
+type in the injected class name. For D}, it matches
+N::M::D of variable d and its class definition and
+injected class name
+D one time respectively.
 </pre></td></tr>
 
 
@@ -2591,8 +3448,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   C c;
   S s;
 
-enumType() matches the type of the variable declarations of both c and
-s.
+
+The matcher enumType() matches the type
+enum C of c ,
+and the type enum S of s .
 </pre></td></tr>
 
 
@@ -2602,9 +3461,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   int (*f)(int);
   void g();
-functionProtoType()
-  matches "int (*f)(int)" and the type of "g" in C++ mode.
-  In C mode, "g" is not matched because it does not contain a prototype.
+The matcher functionProtoType()
+matches the type int (int) of 'f' and the type
+void (void) of 'g' in C++ mode.
+In C, the type void () of 'g' is not
+matched because it does not contain a prototype.
 </pre></td></tr>
 
 
@@ -2614,8 +3475,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   int (*f)(int);
   void g();
-functionType()
-  matches "int (*f)(int)" and the type of "g".
+The matcher functionType()
+int (int) and the type of
+void (void) in C++ and in C23 and
+later. Before C23, the function type for f will be matched the same way,
+but the function type for g will match
+void ().
 </pre></td></tr>
 
 
@@ -2626,27 +3491,30 @@ <h2 id="decl-matchers">Node Matchers</h2>
   int a[] = { 2, 3 };
   int b[42];
   void f(int c[]) { int d[a[0]]; };
-incompleteArrayType()
-  matches "int a[]" and "int c[]"
+The matcher incompleteArrayType()
+int[] and int[]
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('injectedClassNameType0')"><a name="injectedClassNameType0Anchor">injectedClassNameType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1InjectedClassNameType.html">InjectedClassNameType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="injectedClassNameType0"><pre>Matches injected class name types.
 
-Example matches S s, but not S&lt;T&gt; s.
-    (matcher = parmVarDecl(hasType(injectedClassNameType())))
+Given
   template &lt;typename T&gt; struct S {
     void f(S s);
     void g(S&lt;T&gt; s);
   };
+
+The matcher
+parmVarDecl(hasType(elaboratedType(namesType(injectedClassNameType()))))
+matches S s, but not s}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('lValueReferenceType0')"><a name="lValueReferenceType0Anchor">lValueReferenceType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1LValueReferenceType.html">LValueReferenceType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="lValueReferenceType0"><pre>Matches lvalue reference types.
 
-Given:
+Given
   int *a;
   int &amp;b = *a;
   int &amp;&amp;c = 1;
@@ -2655,8 +3523,11 @@ <h2 id="decl-matchers">Node Matchers</h2>
   auto &amp;&amp;f = 2;
   int g = 5;
 
-lValueReferenceType() matches the types of b, d, and e. e is
-matched since the type is deduced as int&amp; by reference collapsing rules.
+
+The matcher lValueReferenceType() matches the type
+int &amp; of b and the type auto &amp;
+of d.
+FIXME: figure out why auto changechange matches twice
 </pre></td></tr>
 
 
@@ -2667,18 +3538,23 @@ <h2 id="decl-matchers">Node Matchers</h2>
   #define CDECL __attribute__((cdecl))
   typedef void (CDECL *X)();
   typedef void (__attribute__((cdecl)) *Y)();
-macroQualifiedType()
-  matches the type of the typedef declaration of X but not Y.
+The matcher macroQualifiedType()
+matches the type CDECL void
+(void) of the typedef declaration of X , unless when in C98-C17, there
+CDECL void (),
+but it does not match the type
+__attribute((cdecl)) void () of Y .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('memberPointerType0')"><a name="memberPointerType0Anchor">memberPointerType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="memberPointerType0"><pre>Matches member pointer types.
 Given
-  struct A { int i; }
-  A::* ptr = A::i;
-memberPointerType()
-  matches "A::* ptr"
+  struct A { int i; };
+  int A::* ptr = &amp;A::i;
+
+The matcher memberPointerType()
+matches int struct A::*.
 </pre></td></tr>
 
 
@@ -2692,8 +3568,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   @interface Foo
   @end
   Foo *f;
-pointerType()
-  matches "Foo *f", but does not match "int *a".
+
+The matcher pointerType()
+matches Foo *, but does not match
+int *.
 </pre></td></tr>
 
 
@@ -2704,8 +3582,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   int (*ptr_to_array)[4];
   int *array_of_ptrs[4];
 
-varDecl(hasType(pointsTo(parenType()))) matches ptr_to_array but not
-array_of_ptrs.
+The matcher varDecl(hasType(pointsTo(parenType())))
+  matches ptr_to_array but not
+  array_of_ptrs.
 </pre></td></tr>
 
 
@@ -2714,22 +3593,28 @@ <h2 id="decl-matchers">Node Matchers</h2>
 types.
 
 Given
-  int *a;
-  int &amp;b = *a;
-  int c = 5;
+  typedef int* int_ptr;
+  void foo(char *str,
+           int val,
+           int *val_ptr,
+           int_ptr not_a_ptr,
+           int_ptr *ptr);
+
+The matcher parmVarDecl(hasType(pointerType()))
+matches char *str, int *val_ptr and
+int_ptr *ptr.
 
   @interface Foo
   @end
   Foo *f;
-pointerType()
-  matches "int *a", but does not match "Foo *f".
+
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('rValueReferenceType0')"><a name="rValueReferenceType0Anchor">rValueReferenceType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1RValueReferenceType.html">RValueReferenceType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="rValueReferenceType0"><pre>Matches rvalue reference types.
 
-Given:
+Given
   int *a;
   int &amp;b = *a;
   int &amp;&amp;c = 1;
@@ -2738,8 +3623,10 @@ <h2 id="decl-matchers">Node Matchers</h2>
   auto &amp;&amp;f = 2;
   int g = 5;
 
-rValueReferenceType() matches the types of c and f. e is not
-matched as it is deduced to int&amp; by reference collapsing rules.
+
+The matcher rValueReferenceType() matches the type
+int &amp;&amp; of c and the type
+auto &amp;&amp; of f.
 </pre></td></tr>
 
 
@@ -2753,8 +3640,14 @@ <h2 id="decl-matchers">Node Matchers</h2>
   C c;
   S s;
 
-recordType() matches the type of the variable declarations of both c
-and s.
+
+The matcher recordType() matches the type
+class C of the variable declaration of c and
+matches the type struct S of the variable
+declaration of s.
+Both of these types are matched three times, once for the type of the
+variable, once for the definition of the class, and once for the type of the
+injected class name.
 </pre></td></tr>
 
 
@@ -2770,7 +3663,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
   auto &amp;&amp;f = 2;
   int g = 5;
 
-referenceType() matches the types of b, c, d, e, and f.
+
+The matcher referenceType() matches the type
+int &amp; of b , the type int &amp;&amp; of
+c, the type
+auto &amp; d, and the type
+auto &amp;&amp; of e and f.
 </pre></td></tr>
 
 
@@ -2781,10 +3679,17 @@ <h2 id="decl-matchers">Node Matchers</h2>
 Given
   template &lt;typename T&gt;
   void F(T t) {
+    T local;
     int i = 1 + t;
   }
+  void f() {
+    F(0);
+  }
 
-substTemplateTypeParmType() matches the type of 't' but not '1'
+
+The matcher varDecl(hasType(substTemplateTypeParmType()))
+matches T t and T local for the substituted template type
+int in the instantiation of F .
 </pre></td></tr>
 
 
@@ -2792,14 +3697,18 @@ <h2 id="decl-matchers">Node Matchers</h2>
 <tr><td colspan="4" class="doc" id="tagType0"><pre>Matches tag types (record and enum types).
 
 Given
-  enum E {};
+  enum E { Ok };
   class C {};
 
   E e;
   C c;
 
-tagType() matches the type of the variable declarations of both e
-and c.
+
+The matcher tagType() matches the type
+enum E of variable e and the type
+class C three times, once for the type
+of the variable c , once for the type of the class definition and once of
+the type in the injected class name.
 </pre></td></tr>
 
 
@@ -2810,25 +3719,38 @@ <h2 id="decl-matchers">Node Matchers</h2>
   template &lt;typename T&gt;
   class C { };
 
-  template class C&lt;int&gt;;  // A
-  C&lt;char&gt; var;            // B
+  template class C&lt;int&gt;;
+  C&lt;int&gt; intvar;
+  C&lt;char&gt; charvar;
 
-templateSpecializationType() matches the type of the explicit
-instantiation in A and the type of the variable declaration in B.
+
+The matcher templateSpecializationType() matches the type
+C&lt;int&gt; of the explicit instantiation in A and the
+type C&lt;char&gt; of the variable declaration in
+B.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('templateTypeParmType0')"><a name="templateTypeParmType0Anchor">templateTypeParmType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateTypeParmType.html">TemplateTypeParmType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="templateTypeParmType0"><pre>Matches template type parameter types.
 
-Example matches T, but not int.
-    (matcher = templateTypeParmType())
+Given
   template &lt;typename T&gt; void f(int i);
+
+The matcher templateTypeParmType() matches T,
+but does not match int.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('type0')"><a name="type0Anchor">type</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="type0"><pre>Matches Types in the clang AST.
+
+Given
+  const int b = 1;
+
+The matcher varDecl(hasType(type().bind("type")))
+matches const int b = 1, with type()
+matching int.
 </pre></td></tr>
 
 
@@ -2837,18 +3759,22 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   typedef int X;
-typedefType()
-  matches "typedef int X"
+  X x = 0;
+The matcher typedefType()
+matches X.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('unaryTransformType0')"><a name="unaryTransformType0Anchor">unaryTransformType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnaryTransformType.html">UnaryTransformType</a>&gt;...</td></tr>
 <tr><td colspan="4" class="doc" id="unaryTransformType0"><pre>Matches types nodes representing unary type transformations.
 
-Given:
-  typedef __underlying_type(T) type;
-unaryTransformType()
-  matches "__underlying_type(T)"
+Given
+  template &lt;typename T&gt; struct A {
+    typedef __underlying_type(T) type;
+  };
+
+The matcher unaryTransformType()
+matches __underlying_type(T)
 </pre></td></tr>
 
 
@@ -2860,7 +3786,9 @@ <h2 id="decl-matchers">Node Matchers</h2>
   using a::S;
   S s;
 
-usingType() matches the type of the variable declaration of s.
+
+The matcher usingType() matches the type a::S
+of the variable declaration of s.
 </pre></td></tr>
 
 
@@ -2870,12 +3798,12 @@ <h2 id="decl-matchers">Node Matchers</h2>
 
 Given
   void f() {
-    int a[] = { 2, 3 }
+    int a[] = { 2, 3 };
     int b[42];
     int c[a[0]];
   }
-variableArrayType()
-  matches "int c[a[0]]"
+The matcher variableArrayType()
+int[a[0]]
 </pre></td></tr>
 
 <!--END_DECL_MATCHERS -->
@@ -2899,6 +3827,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="allOf0"><pre>Matches if all given matchers match.
 
 Usable as: Any Matcher
+
+  int v0 = 0;
+  int v1 = 1;
+
+The matcher varDecl(allOf(hasName("v0"), hasType(isInteger())))
+matches int v0 = 0.
 </pre></td></tr>
 
 
@@ -2906,6 +3840,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="anyOf0"><pre>Matches if any of the given matchers matches.
 
 Usable as: Any Matcher
+
+  char v0 = 'a';
+  int v1 = 1;
+  float v2 = 2.0;
+
+The matcher varDecl(anyOf(hasName("v0"), hasType(isInteger())))
+matches char v0 = 'a' and int v1 = 1.
 </pre></td></tr>
 
 
@@ -2916,11 +3857,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 additional constraint. This will often be used with an explicit conversion
 to an internal::Matcher&lt;&gt; type such as TypeMatcher.
 
-Example: DeclarationMatcher(anything()) matches all declarations, e.g.,
-"int* p" and "void f()" in
+Given
   int* p;
   void f();
-
+The matcher decl(anything())
+matches int* p and void f().
 Usable as: Any Matcher
 </pre></td></tr>
 
@@ -2929,21 +3870,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="mapAnyOf0"><pre>Matches any of the NodeMatchers with InnerMatchers nested within
 
 Given
-  if (true);
-  for (; true; );
-with the matcher
-  mapAnyOf(ifStmt, forStmt).with(
-    hasCondition(cxxBoolLiteralExpr(equals(true)))
-    ).bind("trueCond")
-matches the if and the for. It is equivalent to:
-  auto trueCond = hasCondition(cxxBoolLiteralExpr(equals(true)));
-  anyOf(
-    ifStmt(trueCond).bind("trueCond"),
-    forStmt(trueCond).bind("trueCond")
-    );
+  void f() {
+    if (true);
+    for (; true; );
+  }
+
+
+The matcher stmt(mapAnyOf(ifStmt, forStmt).with(
+    hasCondition(cxxBoolLiteral(equals(true)))
+    )),
+which is equivalent to
+stmt(anyOf(
+    ifStmt(hasCondition(cxxBoolLiteral(equals(true)))).bind("trueCond"),
+    forStmt(hasCondition(cxxBoolLiteral(equals(true)))).bind("trueCond")
+    )),
+matches if (true); and for (; true; );.
 
 The with() chain-call accepts zero or more matchers which are combined
 as-if with allOf() in each of the node matchers.
+
 Usable as: Any Matcher
 </pre></td></tr>
 
@@ -2951,10 +3896,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;*&gt;</td><td class="name" onclick="toggle('unless0')"><a name="unless0Anchor">unless</a></td><td>Matcher&lt;*&gt;</td></tr>
 <tr><td colspan="4" class="doc" id="unless0"><pre>Matches if the provided matcher does not match.
 
-Example matches Y (matcher = cxxRecordDecl(unless(hasName("X"))))
+Given
   class X {};
   class Y {};
 
+The matcher cxxRecordDecl(unless(hasName("X")))
+matches Y
+
 Usable as: Any Matcher
 </pre></td></tr>
 
@@ -2962,6 +3910,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Attr.html">Attr</a>&gt;</td><td class="name" onclick="toggle('isImplicit1')"><a name="isImplicit1Anchor">isImplicit</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isImplicit1"><pre>Matches an entity that has been implicitly added by the compiler (e.g.
 implicit default/copy constructors).
+
+Given
+  struct S {};
+  void f(S obj) {
+    S copy = obj;
+    [&amp;](){ return copy; };
+  }
+
+
+The matcher cxxConstructorDecl(isImplicit(), isCopyConstructor())
+matches the implicit copy constructor of S.
+The matcher lambdaExpr(forEachLambdaCapture(
+    lambdaCapture(isImplicit()))) matches [&amp;](){ return copy; },
+because it implicitly captures copy .
 </pre></td></tr>
 
 
@@ -2969,9 +3931,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyOperatorName0"><pre>Matches operator expressions (binary or unary) that have any of the
 specified names.
 
-   hasAnyOperatorName("+", "-")
- Is equivalent to
-   anyOf(hasOperatorName("+"), hasOperatorName("-"))
+It provides a compact way of writing if an operator has any of the specified
+names:
+The matcher
+   hasAnyOperatorName("+", "-")
+Is equivalent to
+   hasOperatorName("-"))}
+
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+void bar(bool a, bool b) {
+  a &amp;&amp; b;
+ }
+
+The matcher binaryOperator(hasAnyOperatorName("||", "&amp;&amp;"))
+matches a || b and a &amp;&amp; b.
+The matcher unaryOperator(hasAnyOperatorName("-", "!"))
+matches !(a || b).
 </pre></td></tr>
 
 
@@ -2979,43 +3958,62 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperatorName0"><pre>Matches the operator Name of operator expressions and fold expressions
 (binary or unary).
 
-Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
-  !(a || b)
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+The matcher binaryOperator(hasOperatorName("||"))
+matches a || b
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasOperatorName("+")))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+The matcher cxxFoldExpr(hasOperatorName("+"))
+ matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;</td><td class="name" onclick="toggle('isAssignmentOperator0')"><a name="isAssignmentOperator0Anchor">isAssignmentOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isAssignmentOperator0"><pre>Matches all kinds of assignment operators.
 
-Example 1: matches a += b (matcher = binaryOperator(isAssignmentOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isAssignmentOperator())
+matches a += b.
 
-Example 2: matches s1 = s2
-           (matcher = cxxOperatorCallExpr(isAssignmentOperator()))
+Given
   struct S { S&amp; operator=(const S&amp;); };
   void x() { S s1, s2; s1 = s2; }
+
+The matcher cxxOperatorCallExpr(isAssignmentOperator())
+matches s1 = s2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;</td><td class="name" onclick="toggle('isComparisonOperator0')"><a name="isComparisonOperator0Anchor">isComparisonOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isComparisonOperator0"><pre>Matches comparison operators.
 
-Example 1: matches a == b (matcher = binaryOperator(isComparisonOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isComparisonOperator())
+matches a == b
 
-Example 2: matches s1 &lt; s2
-           (matcher = cxxOperatorCallExpr(isComparisonOperator()))
+Given
   struct S { bool operator&lt;(const S&amp; other); };
   void x(S s1, S s2) { bool b1 = s1 &lt; s2; }
+
+The matcher cxxOperatorCallExpr(isComparisonOperator())
+matches s1 &lt; s2
 </pre></td></tr>
 
 
@@ -3023,16 +4021,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isPrivate1"><pre>Matches private C++ declarations and C++ base specifers that specify private
 inheritance.
 
-Examples:
+Given
   class C {
   public:    int a;
   protected: int b;
-  private:   int c; // fieldDecl(isPrivate()) matches 'c'
+  private:   int c;
   };
 
+The matcher fieldDecl(isPrivate())
+matches c.
+
   struct Base {};
-  struct Derived1 : private Base {}; // matches 'Base'
-  class Derived2 : Base {}; // matches 'Base'
+  struct Derived1 : private Base {}; // Base
+  class Derived2 : Base {}; // Base
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPrivate()).bind("base")))
+matches Derived1 and Derived2, with
+cxxBaseSpecifier(isPrivate()) matching
+Base.
 </pre></td></tr>
 
 
@@ -3040,15 +4047,24 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isProtected1"><pre>Matches protected C++ declarations and C++ base specifers that specify
 protected inheritance.
 
-Examples:
+Given
   class C {
   public:    int a;
-  protected: int b; // fieldDecl(isProtected()) matches 'b'
+  protected: int b;
   private:   int c;
   };
 
+The matcher fieldDecl(isProtected())
+matches b.
+
   class Base {};
-  class Derived : protected Base {}; // matches 'Base'
+  class Derived : protected Base {};
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isProtected()).bind("base")))
+matches Derived, with
+cxxBaseSpecifier(isProtected()) matching
+Base.
 </pre></td></tr>
 
 
@@ -3056,16 +4072,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isPublic1"><pre>Matches public C++ declarations and C++ base specifers that specify public
 inheritance.
 
-Examples:
+Given
   class C {
-  public:    int a; // fieldDecl(isPublic()) matches 'a'
+  public:    int a;
   protected: int b;
   private:   int c;
   };
 
+The matcher fieldDecl(isPublic())
+matches a.
+
+Given
   class Base {};
-  class Derived1 : public Base {}; // matches 'Base'
-  struct Derived2 : Base {}; // matches 'Base'
+  class Derived1 : public Base {};
+  struct Derived2 : Base {};
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPublic()).bind("base")))
+matches Derived1 and Derived2,
+with cxxBaseSpecifier(isPublic()) matching
+public Base and Base.
 </pre></td></tr>
 
 
@@ -3073,16 +4099,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isVirtual1"><pre>Matches declarations of virtual methods and C++ base specifers that specify
 virtual inheritance.
 
-Example:
+Given
   class A {
    public:
     virtual void x(); // matches x
   };
 
-Example:
-  class Base {};
-  class DirectlyDerived : virtual Base {}; // matches Base
-  class IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
+The matcher cxxMethodDecl(isVirtual())
+matches x.
+
+Given
+  struct Base {};
+  struct DirectlyDerived : virtual Base {}; // matches Base
+  struct IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
+
+The matcher
+cxxRecordDecl(hasDirectBase(cxxBaseSpecifier(isVirtual())))
+matches DirectlyDerived.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -3096,22 +4129,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="equals2"><pre>Matches literals that are equal to the given value of type ValueT.
 
 Given
+void f(char, bool, double, int);
+void foo() {
   f('false, 3.14, 42);
-characterLiteral(equals(0))
-  matches 'cxxBoolLiteral(equals(false)) and cxxBoolLiteral(equals(0))
-  match false
-floatLiteral(equals(3.14)) and floatLiteral(equals(314e-2))
-  match 3.14
-integerLiteral(equals(42))
-  matches 42
+}
+
+The matcher characterLiteral(equals(0U)) matches 'The matchers cxxBoolLiteral(equals(false)) and
+cxxBoolLiteral(equals(0)) match false.
+The matcher floatLiteral(equals(3.14)) matches 3.14.
+The matcher integerLiteral(equals(42)) matches 42.
 
 Note that you cannot directly match a negative numeric literal because the
 minus sign is not part of the literal: It is a unary operator whose operand
 is the positive numeric literal. Instead, you must use a unaryOperator()
 matcher to match the minus sign:
 
-unaryOperator(hasOperatorName("-"),
-              hasUnaryOperand(integerLiteral(equals(13))))
+Given
+  int val = -1;
+
+The matcher unaryOperator(hasOperatorName("-"),
+              hasUnaryOperand(integerLiteral(equals(1))))
+matches -1.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CharacterLiteral.html">CharacterLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBoolLiteralExpr.html">CXXBoolLiteralExpr</a>&gt;,
            Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FloatingLiteral.html">FloatingLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IntegerLiteral.html">IntegerLiteral</a>&gt;
@@ -3130,14 +4168,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isCatchAll0"><pre>Matches a C++ catch statement that has a catch-all handler.
 
 Given
-  try {
-    // ...
-  } catch (int) {
-    // ...
-  } catch (...) {
-    // ...
+  void foo() {
+    try {}
+    catch (int) {}
+    catch (...) {}
   }
-cxxCatchStmt(isCatchAll()) matches catch(...) but not catch(int).
+
+The matcher cxxCatchStmt(isCatchAll())
+matches catch (...) {}
+but does not match catch(int)
 </pre></td></tr>
 
 
@@ -3145,12 +4184,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountAtLeast1"><pre>Checks that a call expression or a constructor call expression has at least
 the specified number of arguments (including absent default arguments).
 
-Example matches f(0, 0) and g(0, 0, 0)
-(matcher = callExpr(argumentCountAtLeast(2)))
+Given
   void f(int x, int y);
   void g(int x, int y, int z);
-  f(0, 0);
-  g(0, 0, 0);
+  void foo() {
+    f(0, 0);
+    g(0, 0, 0);
+  }
+The matcher callExpr(argumentCountAtLeast(2))
+matches f(0, 0) and g(0, 0, 0)
 </pre></td></tr>
 
 
@@ -3158,14 +4200,39 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountIs1"><pre>Checks that a call expression or a constructor call expression has
 a specific number of arguments (including absent default arguments).
 
-Example matches f(0, 0) (matcher = callExpr(argumentCountIs(2)))
+Given
   void f(int x, int y);
-  f(0, 0);
+  void foo() {
+    f(0, 0);
+  }
+The matcher callExpr(argumentCountIs(2))
+matches f(0, 0)
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;</td><td class="name" onclick="toggle('isListInitialization0')"><a name="isListInitialization0Anchor">isListInitialization</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isListInitialization0"><pre>Matches a constructor call expression which uses list initialization.
+
+Given
+  namespace std {
+    template &lt;typename T&gt;
+    class initializer_list {
+      const T* begin;
+      const T* end;
+    };
+  }
+  template &lt;typename T&gt; class vector {
+    public: vector(std::initializer_list&lt;T&gt;) {}
+  };
+
+  vector&lt;int&gt; a({ 1, 2, 3 });
+  vector&lt;int&gt; b = { 4, 5 };
+  int c[] = { 6, 7 };
+  struct pair { int x; int y; };
+  pair d = { 8, 9 };
+
+The matcher cxxConstructExpr(isListInitialization())
+matches { 4, 5 }.
 </pre></td></tr>
 
 
@@ -3175,11 +4242,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
 void foo() {
-  struct point { double x; double y; };
-  point pt[2] = { { 1.0, 2.0 } };
+  struct Foo {
+    double x;
+  };
+  auto Val = Foo();
 }
-initListExpr(has(cxxConstructExpr(requiresZeroInitialization()))
-will match the implicit array filler for pt[1].
+
+The matcher
+cxxConstructExpr(requiresZeroInitialization())
+matches Foo() because the x member has to be zero initialized.
 </pre></td></tr>
 
 
@@ -3192,7 +4263,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     S(const S &amp;); // #2
     S(S &amp;&amp;); // #3
   };
-cxxConstructorDecl(isCopyConstructor()) will match #2, but not #1 or #3.
+
+The matcher cxxConstructorDecl(isCopyConstructor())
+matches S(const S &amp;),
+but does not match S() or S(S &amp;&amp;).
 </pre></td></tr>
 
 
@@ -3205,7 +4279,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     S(const S &amp;); // #2
     S(S &amp;&amp;); // #3
   };
-cxxConstructorDecl(isDefaultConstructor()) will match #1, but not #2 or #3.
+
+The matcher cxxConstructorDecl(isDefaultConstructor())
+matches S()
+but does not match S(const S &amp;); or S(S &amp;&amp;);.
 </pre></td></tr>
 
 
@@ -3219,8 +4296,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     S(S &amp;&amp;) : S() {} // #3
   };
   S::S() : S(0) {} // #4
-cxxConstructorDecl(isDelegatingConstructor()) will match #3 and #4, but not
-#1 or #2.
+
+The matcher cxxConstructorDecl(isDelegatingConstructor())
+matches S(S &amp;&amp;) : S() {} and S::S() : S(0) {},
+but does not match S() or S(int).
 </pre></td></tr>
 
 
@@ -3236,15 +4315,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     explicit S(double); // #2
     operator int(); // #3
     explicit operator bool(); // #4
-    explicit(false) S(bool) // # 7
-    explicit(true) S(char) // # 8
-    explicit(b) S(S) // # 9
+    explicit(false) S(bool); // # 7
+    explicit(true) S(char); // # 8
+    explicit(b) S(float); // # 9
   };
-  S(int) -&gt; S&lt;true&gt; // #5
-  explicit S(double) -&gt; S&lt;false&gt; // #6
-cxxConstructorDecl(isExplicit()) will match #2 and #8, but not #1, #7 or #9.
-cxxConversionDecl(isExplicit()) will match #4, but not #3.
-cxxDeductionGuideDecl(isExplicit()) will match #6, but not #5.
+  S(int) -&gt; S&lt;true&gt;; // #5
+  explicit S(double) -&gt; S&lt;false&gt;; // #6
+
+The matcher cxxConstructorDecl(isExplicit())
+matches explicit S(double)
+and explicit(true) S(char)
+but does not match S(int);, explicit(false) S(bool); or
+explicit(b) S(float)
+The matcher cxxConversionDecl(isExplicit())
+matches explicit operator bool()
+but does not match operator int().
+The matcher cxxDeductionGuideDecl(isExplicit())
+matches the deduction guide explicit S(double) -&gt; S&lt;false&gt;,
+the implicit copy deduction candiate
+auto (double) -&gt; S&lt;b&gt; and
+the implicitly generated deduction guide for explicit(true) S(char),
+but does not match S(int) -&gt; S&lt;true&gt;.
 </pre></td></tr>
 
 
@@ -3261,7 +4352,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     S(const S &amp;); // #2
     S(S &amp;&amp;); // #3
   };
-cxxConstructorDecl(isMoveConstructor()) will match #3, but not #1 or #2.
+
+The matcher cxxConstructorDecl(isMoveConstructor())
+matches S(S &amp;&amp;)
+but does not match S(); or S(S &amp;&amp;);
 </pre></td></tr>
 
 
@@ -3277,15 +4371,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     explicit S(double); // #2
     operator int(); // #3
     explicit operator bool(); // #4
-    explicit(false) S(bool) // # 7
-    explicit(true) S(char) // # 8
-    explicit(b) S(S) // # 9
+    explicit(false) S(bool); // # 7
+    explicit(true) S(char); // # 8
+    explicit(b) S(float); // # 9
   };
-  S(int) -&gt; S&lt;true&gt; // #5
-  explicit S(double) -&gt; S&lt;false&gt; // #6
-cxxConstructorDecl(isExplicit()) will match #2 and #8, but not #1, #7 or #9.
-cxxConversionDecl(isExplicit()) will match #4, but not #3.
-cxxDeductionGuideDecl(isExplicit()) will match #6, but not #5.
+  S(int) -&gt; S&lt;true&gt;; // #5
+  explicit S(double) -&gt; S&lt;false&gt;; // #6
+
+The matcher cxxConstructorDecl(isExplicit())
+matches explicit S(double)
+and explicit(true) S(char)
+but does not match S(int);, explicit(false) S(bool); or
+explicit(b) S(float)
+The matcher cxxConversionDecl(isExplicit())
+matches explicit operator bool()
+but does not match operator int().
+The matcher cxxDeductionGuideDecl(isExplicit())
+matches the deduction guide explicit S(double) -&gt; S&lt;false&gt;,
+the implicit copy deduction candiate
+auto (double) -&gt; S&lt;b&gt; and
+the implicitly generated deduction guide for explicit(true) S(char),
+but does not match S(int) -&gt; S&lt;true&gt;.
 </pre></td></tr>
 
 
@@ -3302,8 +4408,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   struct E : B {
     E() : B() {}
   };
+
+The matcher
 cxxConstructorDecl(hasAnyConstructorInitializer(isBaseInitializer()))
-  will match E(), but not match D(int).
+matches E() : B() {} and D(int i) : I(i) {}.
+The constructor of D is matched, because it implicitly has a constructor
+initializer for B .
 </pre></td></tr>
 
 
@@ -3320,8 +4430,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   struct E : B {
     E() : B() {}
   };
+
+The matcher
 cxxConstructorDecl(hasAnyConstructorInitializer(isMemberInitializer()))
-  will match D(int), but not match E().
+  will match D(int i) : I(i) {}, but not match E() : B()
+  {}.
 </pre></td></tr>
 
 
@@ -3330,13 +4443,16 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 code (as opposed to implicitly added by the compiler).
 
 Given
+  struct Bar { explicit Bar(const char*); };
   struct Foo {
     Foo() { }
     Foo(int) : foo_("A") { }
-    string foo_;
+    Bar foo_{""};
   };
-cxxConstructorDecl(hasAnyConstructorInitializer(isWritten()))
-  will match Foo(int), but not Foo()
+
+The matcher
+cxxConstructorDecl(hasAnyConstructorInitializer(isWritten())) will
+match Foo(int) : foo_("A") { }, but not Foo() { }
 </pre></td></tr>
 
 
@@ -3352,15 +4468,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     explicit S(double); // #2
     operator int(); // #3
     explicit operator bool(); // #4
-    explicit(false) S(bool) // # 7
-    explicit(true) S(char) // # 8
-    explicit(b) S(S) // # 9
+    explicit(false) S(bool); // # 7
+    explicit(true) S(char); // # 8
+    explicit(b) S(float); // # 9
   };
-  S(int) -&gt; S&lt;true&gt; // #5
-  explicit S(double) -&gt; S&lt;false&gt; // #6
-cxxConstructorDecl(isExplicit()) will match #2 and #8, but not #1, #7 or #9.
-cxxConversionDecl(isExplicit()) will match #4, but not #3.
-cxxDeductionGuideDecl(isExplicit()) will match #6, but not #5.
+  S(int) -&gt; S&lt;true&gt;; // #5
+  explicit S(double) -&gt; S&lt;false&gt;; // #6
+
+The matcher cxxConstructorDecl(isExplicit())
+matches explicit S(double)
+and explicit(true) S(char)
+but does not match S(int);, explicit(false) S(bool); or
+explicit(b) S(float)
+The matcher cxxConversionDecl(isExplicit())
+matches explicit operator bool()
+but does not match operator int().
+The matcher cxxDeductionGuideDecl(isExplicit())
+matches the deduction guide explicit S(double) -&gt; S&lt;false&gt;,
+the implicit copy deduction candiate
+auto (double) -&gt; S&lt;b&gt; and
+the implicitly generated deduction guide for explicit(true) S(char),
+but does not match S(int) -&gt; S&lt;true&gt;.
 </pre></td></tr>
 
 
@@ -3382,7 +4510,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
       S&lt;T&gt; s;
       s.mem();
   }
-cxxDependentScopeMemberExpr(hasMemberName("mem")) matches `s.mem()`
+
+The matcher cxxDependentScopeMemberExpr(hasMemberName("mem"))
+matches s.mem.
 </pre></td></tr>
 
 
@@ -3401,14 +4531,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   };
   template &lt;class T&gt;
   class Z {
-    void x() { this-&gt;m; }
+    void x() {
+      this-&gt;m;
+      this-&gt;t;
+      this-&gt;t-&gt;m;
+    }
+    int m;
+    T* t;
   };
-memberExpr(isArrow())
-  matches this-&gt;x, x, y.x, a, this-&gt;b
-cxxDependentScopeMemberExpr(isArrow())
-  matches this-&gt;m
-unresolvedMemberExpr(isArrow())
-  matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
+
+The matcher memberExpr(isArrow())
+matches this-&gt;x, x, a,
+this-&gt;b, this-&gt;m and two times this-&gt;t,
+once for the standalone member expression, and once for the member
+expression that later accesses m .
+Additionally, it does not match this-&gt;t-&gt;t.
+The matcher cxxDependentScopeMemberExpr(isArrow())
+matches this-&gt;t-&gt;m, but not this-&gt;m or this-&gt;t.
+The matcher unresolvedMemberExpr(isArrow())
+matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
 </pre></td></tr>
 
 
@@ -3432,19 +4573,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
       S&lt;T&gt; s;
       s.mem();
   }
-The matcher
-@code
-cxxDependentScopeMemberExpr(
-  hasObjectExpression(declRefExpr(hasType(templateSpecializationType(
+
+The matcher cxxDependentScopeMemberExpr(
+  hasObjectExpression(declRefExpr(hasType(
+    elaboratedType(namesType(templateSpecializationType(
       hasDeclaration(classTemplateDecl(has(cxxRecordDecl(has(
           cxxMethodDecl(hasName("mem")).bind("templMem")
           )))))
-      )))),
+    )))
+  ))),
   memberHasSameNameAsBoundNode("templMem")
-  )
-@endcode
-first matches and binds the @c mem member of the @c S template, then
-compares its name to the usage in @c s.mem() in the @c x function template
+)
+matches s.mem, with the inner matcher
+cxxMethodDecl(hasName("mem")) matching
+void mem() of the S template.
 </pre></td></tr>
 
 
@@ -3452,23 +4594,29 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperatorName3"><pre>Matches the operator Name of operator expressions and fold expressions
 (binary or unary).
 
-Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
-  !(a || b)
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+The matcher binaryOperator(hasOperatorName("||"))
+matches a || b
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasOperatorName("+")))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+The matcher cxxFoldExpr(hasOperatorName("+"))
+ matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('isBinaryFold0')"><a name="isBinaryFold0Anchor">isBinaryFold</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isBinaryFold0"><pre>Matches binary fold expressions, i.e. fold expressions with an initializer.
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(isBinaryFold()))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -3478,14 +4626,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   auto multiply(Args... args) {
       return (args * ...);
   }
+
+
+The matcher cxxFoldExpr(isBinaryFold())
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('isLeftFold0')"><a name="isLeftFold0Anchor">isLeftFold</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isLeftFold0"><pre>Matches left-folding fold expressions.
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(isLeftFold()))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -3495,14 +4646,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
+
+
+The matcher cxxFoldExpr(isLeftFold())
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('isRightFold0')"><a name="isRightFold0Anchor">isRightFold</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isRightFold0"><pre>Matches right-folding fold expressions.
 
-Example matches `(args * ... * 1)`
-    (matcher = cxxFoldExpr(isRightFold()))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -3512,6 +4666,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
+
+
+The matcher cxxFoldExpr(isRightFold())
+matches (args * ... * 1).
 </pre></td></tr>
 
 
@@ -3519,8 +4677,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isUnaryFold0"><pre>Matches unary fold expressions, i.e. fold expressions without an
 initializer.
 
-Example matches `(args * ...)`
-    (matcher = cxxFoldExpr(isUnaryFold()))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -3530,6 +4687,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   auto multiply(Args... args) {
       return (args * ...);
   }
+
+
+The matcher cxxFoldExpr(isUnaryFold())
+matches (args * ...), but not (0 + ... + args).
 </pre></td></tr>
 
 
@@ -3542,7 +4703,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void bar();
 };
 
-cxxMethodDecl(isConst()) matches A::foo() but not A::bar()
+
+The matcher cxxMethodDecl(isConst())
+matches foo but not bar
 </pre></td></tr>
 
 
@@ -3556,8 +4719,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   A &amp;operator=(A &amp;&amp;);
 };
 
-cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
-the second one.
+
+The matcher cxxMethodDecl(isCopyAssignmentOperator())
+matches A &amp;operator=(const A &amp;)
+but does not match A &amp;operator=(A &amp;&amp;)
 </pre></td></tr>
 
 
@@ -3573,15 +4738,19 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
  int operator+(int);
 };
 
-cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
-methods but not the last two.
+
+The matcher cxxMethodDecl(isExplicitObjectMemberFunction())
+matches int operator-(this A, int) and
+void fun(this A &amp;&amp;self),
+but not static int operator()(int) or
+int operator+(int).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isFinal1')"><a name="isFinal1Anchor">isFinal</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isFinal1"><pre>Matches if the given method or class declaration is final.
 
-Given:
+Given
   class A final {};
 
   struct B {
@@ -3591,7 +4760,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   struct C : B {
     void f() final;
   };
-matches A and C::f, but not B, C, or B::f
+
+The matcher cxxRecordDecl(isFinal())
+matches A,
+but does not match B or C.
+The matcher cxxMethodDecl(isFinal())
+matches void f() final in C ,
+but does not match virtual void f() in B .
 </pre></td></tr>
 
 
@@ -3605,8 +4780,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   A &amp;operator=(A &amp;&amp;);
 };
 
-cxxMethodDecl(isMoveAssignmentOperator()) matches the second method but not
-the first one.
+
+The matcher cxxMethodDecl(isMoveAssignmentOperator())
+matches A &amp;operator=(A &amp;&amp;)
+but does not match A &amp;operator=(const A &amp;)
 </pre></td></tr>
 
 
@@ -3620,9 +4797,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   };
   class B : public A {
    public:
-    virtual void x();
+    void x() override;
   };
-  matches B::x
+
+The matcher cxxMethodDecl(isOverride())
+  matches void x() override
 </pre></td></tr>
 
 
@@ -3634,7 +4813,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
    public:
     virtual void x() = 0;
   };
-  matches A::x
+
+The matcher cxxMethodDecl(isPure())
+matches virtual void x() = 0
 </pre></td></tr>
 
 
@@ -3647,7 +4828,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     S(const S &amp;) = default; // #2
     S(S &amp;&amp;) = delete; // #3
   };
-cxxConstructorDecl(isUserProvided()) will match #1, but not #2 or #3.
+
+The matcher cxxConstructorDecl(isUserProvided())
+will match S(), but not S &amp;) = default} or
+&amp;&amp;) = delete}
 </pre></td></tr>
 
 
@@ -3655,16 +4839,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isVirtual0"><pre>Matches declarations of virtual methods and C++ base specifers that specify
 virtual inheritance.
 
-Example:
+Given
   class A {
    public:
     virtual void x(); // matches x
   };
 
-Example:
-  class Base {};
-  class DirectlyDerived : virtual Base {}; // matches Base
-  class IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
+The matcher cxxMethodDecl(isVirtual())
+matches x.
+
+Given
+  struct Base {};
+  struct DirectlyDerived : virtual Base {}; // matches Base
+  struct IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
+
+The matcher
+cxxRecordDecl(hasDirectBase(cxxBaseSpecifier(isVirtual())))
+matches DirectlyDerived.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -3682,17 +4873,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
    public:
     void x();
   };
-  matches A::x but not B::x
+
+The matcher cxxMethodDecl(isVirtualAsWritten())
+matches virtual void x() of A,
+but does not match x()} of B .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('isArray0')"><a name="isArray0Anchor">isArray</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isArray0"><pre>Matches array new expressions.
 
-Given:
+Given
+  struct MyClass { int x; };
   MyClass *p1 = new MyClass[10];
-cxxNewExpr(isArray())
-  matches the expression 'new MyClass[10]'.
+
+The matcher cxxNewExpr(isArray())
+matches new MyClass[10].
 </pre></td></tr>
 
 
@@ -3700,9 +4896,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyOperatorName1"><pre>Matches operator expressions (binary or unary) that have any of the
 specified names.
 
+It provides a compact way of writing if an operator has any of the specified
+names:
+The matcher
    hasAnyOperatorName("+", "-")
- Is equivalent to
-   anyOf(hasOperatorName("+"), hasOperatorName("-"))
+Is equivalent to
+   hasOperatorName("-"))}
+
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+void bar(bool a, bool b) {
+  a &amp;&amp; b;
+ }
+
+The matcher binaryOperator(hasAnyOperatorName("||", "&amp;&amp;"))
+matches a || b and a &amp;&amp; b.
+The matcher unaryOperator(hasAnyOperatorName("-", "!"))
+matches !(a || b).
 </pre></td></tr>
 
 
@@ -3713,6 +4926,30 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 "operator" prefix: e.g. "&lt;&lt;".
 
   hasAnyOverloadedOperatorName("+", "-")
+
+Given
+  struct Point { double x; double y; };
+  Point operator+(const Point&amp;, const Point&amp;);
+  Point operator-(const Point&amp;, const Point&amp;);
+
+  Point sub(Point a, Point b) {
+    return b - a;
+  }
+
+
+The matcher functionDecl(hasAnyOverloadedOperatorName("+", "-")),
+which is equivalent to
+functionDecl(anyOf(hasAnyOverloadedOperatorName("+"),
+hasOverloadedOperatorName("-"))),
+matches Point operator+(const Point&amp;, const Point&amp;) and
+Point operator-(const Point&amp;, const Point&amp;).
+The matcher
+cxxOperatorCallExpr(hasAnyOverloadedOperatorName("+", "-")),
+which is equivalent to
+cxxOperatorCallExpr(anyOf(hasOverloadedOperatorName("+"),
+hasOverloadedOperatorName("-"))),
+matches b - a.
+
 Is equivalent to
   anyOf(hasOverloadedOperatorName("+"), hasOverloadedOperatorName("-"))
 </pre></td></tr>
@@ -3722,15 +4959,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperatorName1"><pre>Matches the operator Name of operator expressions and fold expressions
 (binary or unary).
 
-Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
-  !(a || b)
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+The matcher binaryOperator(hasOperatorName("||"))
+matches a || b
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasOperatorName("+")))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+The matcher cxxFoldExpr(hasOperatorName("+"))
+ matches (0 + ... + args).
 </pre></td></tr>
 
 
@@ -3740,16 +4984,19 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Matches overloaded operator names specified in strings without the
 "operator" prefix: e.g. "&lt;&lt;".
 
-Given:
-  class A { int operator*(); };
+Given
+  struct A { int operator*(); };
   const A &amp;operator&lt;&lt;(const A &amp;a, const A &amp;b);
-  A a;
-  a &lt;&lt; a;   // &lt;-- This matches
+  void f(A a) {
+    a &lt;&lt; a;   // &lt;-- This matches
+  }
 
-cxxOperatorCallExpr(hasOverloadedOperatorName("&lt;&lt;"))) matches the
-specified line and
+
+The matcher cxxOperatorCallExpr(hasOverloadedOperatorName("&lt;&lt;"))
+matches a &lt;&lt; a.
+The matcher
 cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")))
-matches the declaration of A.
+matches struct A { int operator*(); }.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;
 </pre></td></tr>
@@ -3758,47 +5005,104 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('isAssignmentOperator1')"><a name="isAssignmentOperator1Anchor">isAssignmentOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isAssignmentOperator1"><pre>Matches all kinds of assignment operators.
 
-Example 1: matches a += b (matcher = binaryOperator(isAssignmentOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isAssignmentOperator())
+matches a += b.
 
-Example 2: matches s1 = s2
-           (matcher = cxxOperatorCallExpr(isAssignmentOperator()))
+Given
   struct S { S&amp; operator=(const S&amp;); };
   void x() { S s1, s2; s1 = s2; }
+
+The matcher cxxOperatorCallExpr(isAssignmentOperator())
+matches s1 = s2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('isComparisonOperator1')"><a name="isComparisonOperator1Anchor">isComparisonOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isComparisonOperator1"><pre>Matches comparison operators.
 
-Example 1: matches a == b (matcher = binaryOperator(isComparisonOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isComparisonOperator())
+matches a == b
 
-Example 2: matches s1 &lt; s2
-           (matcher = cxxOperatorCallExpr(isComparisonOperator()))
+Given
   struct S { bool operator&lt;(const S&amp; other); };
   void x(S s1, S s2) { bool b1 = s1 &lt; s2; }
+
+The matcher cxxOperatorCallExpr(isComparisonOperator())
+matches s1 &lt; s2
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasDefinition0')"><a name="hasDefinition0Anchor">hasDefinition</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasDefinition0"><pre>Matches a class declaration that is defined.
 
-Example matches x (matcher = cxxRecordDecl(hasDefinition()))
+Given
 class x {};
 class y;
+
+The matcher cxxRecordDecl(hasDefinition())
+matches class x {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isDerivedFrom2')"><a name="isDerivedFrom2Anchor">isDerivedFrom</a></td><td>std::string BaseName</td></tr>
 <tr><td colspan="4" class="doc" id="isDerivedFrom2"><pre>Overloaded method as shortcut for isDerivedFrom(hasName(...)).
+
+Matches C++ classes that are directly or indirectly derived from a class
+matching Base, or Objective-C classes that directly or indirectly
+subclass a class matching Base.
+
+Note that a class is not considered to be derived from itself.
+
+Example matches Y, Z, C (Base == hasName("X"))
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+  class Foo {};
+  typedef Foo Alias;
+  class Bar : public Alias {};  // derived from Alias, which is a
+                                // typedef of Foo
+
+
+The matcher cxxRecordDecl(isDerivedFrom("X"))
+matches Y, Z and C.
+The matcher cxxRecordDecl(isDerivedFrom("Foo"))
+matches Bar.
+
+In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
+  @interface NSObject @end
+  @interface Bar : NSObject @end
+
+
+Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isDirectlyDerivedFrom2')"><a name="isDirectlyDerivedFrom2Anchor">isDirectlyDerivedFrom</a></td><td>std::string BaseName</td></tr>
 <tr><td colspan="4" class="doc" id="isDirectlyDerivedFrom2"><pre>Overloaded method as shortcut for isDirectlyDerivedFrom(hasName(...)).
+
+Given
+  struct Base {};
+  struct DirectlyDerived : public Base {};
+  struct IndirectlyDerived : public DirectlyDerived {};
+
+
+The matcher cxxRecordDecl(isDirectlyDerivedFrom("Base"))
+matches DirectlyDerived, but not
+IndirectlyDerived.
 </pre></td></tr>
 
 
@@ -3809,8 +5113,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; void A(T t) { }
   template&lt;&gt; void A(int N) { }
-functionDecl(isExplicitTemplateSpecialization())
-  matches the specialization A&lt;int&gt;().
+
+The matcher functionDecl(isExplicitTemplateSpecialization())
+  matches the specialization template&lt;&gt; void A(int N) { }.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
 </pre></td></tr>
@@ -3819,7 +5124,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isFinal0')"><a name="isFinal0Anchor">isFinal</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isFinal0"><pre>Matches if the given method or class declaration is final.
 
-Given:
+Given
   class A final {};
 
   struct B {
@@ -3829,24 +5134,46 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   struct C : B {
     void f() final;
   };
-matches A and C::f, but not B, C, or B::f
+
+The matcher cxxRecordDecl(isFinal())
+matches A,
+but does not match B or C.
+The matcher cxxMethodDecl(isFinal())
+matches void f() final in C ,
+but does not match virtual void f() in B .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isLambda0')"><a name="isLambda0Anchor">isLambda</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isLambda0"><pre>Matches the generated class of lambda expressions.
 
-Given:
+Given
   auto x = []{};
 
-cxxRecordDecl(isLambda()) matches the implicit class declaration of
-decltype(x)
+
+The matcher varDecl(hasType(cxxRecordDecl(isLambda())))
+matches auto x = []{}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isSameOrDerivedFrom2')"><a name="isSameOrDerivedFrom2Anchor">isSameOrDerivedFrom</a></td><td>std::string BaseName</td></tr>
-<tr><td colspan="4" class="doc" id="isSameOrDerivedFrom2"><pre>Overloaded method as shortcut for
+<tr><td colspan="4" class="doc" id="isSameOrDerivedFrom2"><pre>Similar to isDerivedFrom(), but also matches classes that directly
+match Base.
+Overloaded method as shortcut for
 isSameOrDerivedFrom(hasName(...)).
+
+Given
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+The matcher
+cxxRecordDecl(isSameOrDerivedFrom("X"), isDefinition())
+matches class X {}, class Y : public X {},
+class Z : public Y {} and class C : public B {}.
 </pre></td></tr>
 
 
@@ -3855,18 +5182,36 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 member variable template instantiations.
 
 Given
-  template &lt;typename T&gt; class X {}; class A {}; X&lt;A&gt; x;
-or
-  template &lt;typename T&gt; class X {}; class A {}; template class X&lt;A&gt;;
-or
-  template &lt;typename T&gt; class X {}; class A {}; extern template class X&lt;A&gt;;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
-  matches the template instantiation of X&lt;A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches class X&lt;class A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches template class X&lt;A&gt;
+  template &lt;typename T&gt; class X {};
+  class A {};
+  extern template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches extern template class X&lt;A&gt;
 
 But given
-  template &lt;typename T&gt;  class X {}; class A {};
-  template &lt;&gt; class X&lt;A&gt; {}; X&lt;A&gt; x;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
+  template &lt;typename T&gt;  class X {};
+  class A {};
+  template &lt;&gt; class X&lt;A&gt; {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
   does not match, as X&lt;A&gt; is an explicit template specialization.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
@@ -3877,9 +5222,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyOperatorName2"><pre>Matches operator expressions (binary or unary) that have any of the
 specified names.
 
+It provides a compact way of writing if an operator has any of the specified
+names:
+The matcher
    hasAnyOperatorName("+", "-")
- Is equivalent to
-   anyOf(hasOperatorName("+"), hasOperatorName("-"))
+Is equivalent to
+   hasOperatorName("-"))}
+
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+void bar(bool a, bool b) {
+  a &amp;&amp; b;
+ }
+
+The matcher binaryOperator(hasAnyOperatorName("||", "&amp;&amp;"))
+matches a || b and a &amp;&amp; b.
+The matcher unaryOperator(hasAnyOperatorName("-", "!"))
+matches !(a || b).
 </pre></td></tr>
 
 
@@ -3887,43 +5249,62 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperatorName2"><pre>Matches the operator Name of operator expressions and fold expressions
 (binary or unary).
 
-Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
-  !(a || b)
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+The matcher binaryOperator(hasOperatorName("||"))
+matches a || b
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasOperatorName("+")))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+The matcher cxxFoldExpr(hasOperatorName("+"))
+ matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRewrittenBinaryOperator.html">CXXRewrittenBinaryOperator</a>&gt;</td><td class="name" onclick="toggle('isAssignmentOperator2')"><a name="isAssignmentOperator2Anchor">isAssignmentOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isAssignmentOperator2"><pre>Matches all kinds of assignment operators.
 
-Example 1: matches a += b (matcher = binaryOperator(isAssignmentOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isAssignmentOperator())
+matches a += b.
 
-Example 2: matches s1 = s2
-           (matcher = cxxOperatorCallExpr(isAssignmentOperator()))
+Given
   struct S { S&amp; operator=(const S&amp;); };
   void x() { S s1, s2; s1 = s2; }
+
+The matcher cxxOperatorCallExpr(isAssignmentOperator())
+matches s1 = s2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRewrittenBinaryOperator.html">CXXRewrittenBinaryOperator</a>&gt;</td><td class="name" onclick="toggle('isComparisonOperator2')"><a name="isComparisonOperator2Anchor">isComparisonOperator</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isComparisonOperator2"><pre>Matches comparison operators.
 
-Example 1: matches a == b (matcher = binaryOperator(isComparisonOperator()))
+Given
+void foo(int a, int b) {
   if (a == b)
     a += b;
+}
+The matcher binaryOperator(isComparisonOperator())
+matches a == b
 
-Example 2: matches s1 &lt; s2
-           (matcher = cxxOperatorCallExpr(isComparisonOperator()))
+Given
   struct S { bool operator&lt;(const S&amp; other); };
   void x(S s1, S s2) { bool b1 = s1 &lt; s2; }
+
+The matcher cxxOperatorCallExpr(isComparisonOperator())
+matches s1 &lt; s2
 </pre></td></tr>
 
 
@@ -3931,12 +5312,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountAtLeast2"><pre>Checks that a call expression or a constructor call expression has at least
 the specified number of arguments (including absent default arguments).
 
-Example matches f(0, 0) and g(0, 0, 0)
-(matcher = callExpr(argumentCountAtLeast(2)))
+Given
   void f(int x, int y);
   void g(int x, int y, int z);
-  f(0, 0);
-  g(0, 0, 0);
+  void foo() {
+    f(0, 0);
+    g(0, 0, 0);
+  }
+The matcher callExpr(argumentCountAtLeast(2))
+matches f(0, 0) and g(0, 0, 0)
 </pre></td></tr>
 
 
@@ -3944,9 +5328,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountIs2"><pre>Checks that a call expression or a constructor call expression has
 a specific number of arguments (including absent default arguments).
 
-Example matches f(0, 0) (matcher = callExpr(argumentCountIs(2)))
+Given
   void f(int x, int y);
-  f(0, 0);
+  void foo() {
+    f(0, 0);
+  }
+The matcher callExpr(argumentCountIs(2))
+matches f(0, 0)
 </pre></td></tr>
 
 
@@ -3954,12 +5342,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountAtLeast0"><pre>Checks that a call expression or a constructor call expression has at least
 the specified number of arguments (including absent default arguments).
 
-Example matches f(0, 0) and g(0, 0, 0)
-(matcher = callExpr(argumentCountAtLeast(2)))
+Given
   void f(int x, int y);
   void g(int x, int y, int z);
-  f(0, 0);
-  g(0, 0, 0);
+  void foo() {
+    f(0, 0);
+    g(0, 0, 0);
+  }
+The matcher callExpr(argumentCountAtLeast(2))
+matches f(0, 0) and g(0, 0, 0)
 </pre></td></tr>
 
 
@@ -3967,16 +5358,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountIs0"><pre>Checks that a call expression or a constructor call expression has
 a specific number of arguments (including absent default arguments).
 
-Example matches f(0, 0) (matcher = callExpr(argumentCountIs(2)))
+Given
   void f(int x, int y);
-  f(0, 0);
+  void foo() {
+    f(0, 0);
+  }
+The matcher callExpr(argumentCountIs(2))
+matches f(0, 0)
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;</td><td class="name" onclick="toggle('usesADL0')"><a name="usesADL0Anchor">usesADL</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="usesADL0"><pre>Matches call expressions which were resolved using ADL.
 
-Example matches y(x) but not y(42) or NS::y(x).
+Given
   namespace NS {
     struct X {};
     void y(X);
@@ -3992,15 +5387,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     using NS::y;
     y(x); // Found by both unqualified lookup and ADL, doesn't match
    }
+
+
+The matcher callExpr(usesADL())
+matches y(x), but not y(42) or NS::y(x).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CastExpr.html">CastExpr</a>&gt;</td><td class="name" onclick="toggle('hasCastKind0')"><a name="hasCastKind0Anchor">hasCastKind</a></td><td>CastKind Kind</td></tr>
 <tr><td colspan="4" class="doc" id="hasCastKind0"><pre>Matches casts that has a given cast kind.
 
-Example: matches the implicit cast around 0
-(matcher = castExpr(hasCastKind(CK_NullToPointer)))
+Given
   int *p = 0;
+The matcher castExpr(hasCastKind(CK_NullToPointer))
+matches the implicit cast around 0
 
 If the matcher is use from clang-query, CastKind parameter
 should be passed as a quoted string. e.g., hasCastKind("CK_NullToPointer").
@@ -4015,22 +5415,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="equals3"><pre>Matches literals that are equal to the given value of type ValueT.
 
 Given
+void f(char, bool, double, int);
+void foo() {
   f('false, 3.14, 42);
-characterLiteral(equals(0))
-  matches 'cxxBoolLiteral(equals(false)) and cxxBoolLiteral(equals(0))
-  match false
-floatLiteral(equals(3.14)) and floatLiteral(equals(314e-2))
-  match 3.14
-integerLiteral(equals(42))
-  matches 42
+}
+
+The matcher characterLiteral(equals(0U)) matches 'The matchers cxxBoolLiteral(equals(false)) and
+cxxBoolLiteral(equals(0)) match false.
+The matcher floatLiteral(equals(3.14)) matches 3.14.
+The matcher integerLiteral(equals(42)) matches 42.
 
 Note that you cannot directly match a negative numeric literal because the
 minus sign is not part of the literal: It is a unary operator whose operand
 is the positive numeric literal. Instead, you must use a unaryOperator()
 matcher to match the minus sign:
 
-unaryOperator(hasOperatorName("-"),
-              hasUnaryOperand(integerLiteral(equals(13))))
+Given
+  int val = -1;
+
+The matcher unaryOperator(hasOperatorName("-"),
+              hasUnaryOperand(integerLiteral(equals(1))))
+matches -1.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CharacterLiteral.html">CharacterLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBoolLiteralExpr.html">CXXBoolLiteralExpr</a>&gt;,
            Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FloatingLiteral.html">FloatingLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IntegerLiteral.html">IntegerLiteral</a>&gt;
@@ -4051,8 +5456,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
+
+The matcher
 classTemplateSpecializationDecl(templateArgumentCountIs(1))
-  matches C&lt;int&gt;.
+matches struct C&lt;int&gt;.
 </pre></td></tr>
 
 
@@ -4061,9 +5468,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 child statements.
 
 Example: Given
+void foo() {
   { for (;;) {} }
-compoundStmt(statementCountIs(0)))
-  matches '{}'
+}
+The matcher compoundStmt(statementCountIs(0))
+{}
   but does not match the outer compound statement.
 </pre></td></tr>
 
@@ -4078,10 +5487,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   char *s = "abcd";
   wchar_t *ws = L"abcd";
   char *w = "a";
-constantArrayType(hasSize(42))
-  matches "int a[42]" and "int b[2 * 21]"
-stringLiteral(hasSize(4))
-  matches "abcd", L"abcd"
+
+The matcher constantArrayType(hasSize(42))
+matches int[42] twice.
+The matcher stringLiteral(hasSize(4))
+matches "abcd" and L"abcd".
 </pre></td></tr>
 
 
@@ -4089,12 +5499,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="declCountIs0"><pre>Matches declaration statements that contain a specific number of
 declarations.
 
-Example: Given
-  int a, b;
-  int c;
-  int d = 2, e;
-declCountIs(2)
-  matches 'int a, b;' and 'int d = 2, e;', but not 'int c;'.
+Given
+  void foo() {
+    int a, b;
+    int c;
+    int d = 2, e;
+  }
+The matcher declStmt(declCountIs(2))
+matches int a, b; and int d = 2, e;,
+but does not match int c;
 </pre></td></tr>
 
 
@@ -4105,10 +5518,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   class X { int a; int b; };
-cxxRecordDecl(
+
+The matcher cxxRecordDecl(
     has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))
-  matches the class X, as a and b have the same type.
+  matches X, as a and b have the same type.
 
 Note that when multiple matches are involved via forEach* matchers,
 equalsBoundNodes acts as a filter.
@@ -4121,7 +5535,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('equalsNode0')"><a name="equalsNode0Anchor">equalsNode</a></td><td>const Decl* Other</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('equalsNode0')"><a name="equalsNode0Anchor">equalsNode</a></td><td>const Decl * Other</td></tr>
 <tr><td colspan="4" class="doc" id="equalsNode0"><pre>Matches if a node equals another node.
 
 Decl has pointer identity in the AST.
@@ -4132,10 +5546,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAttr0"><pre>Matches declaration that has a given attribute.
 
 Given
-  __attribute__((device)) void f() { ... }
-decl(hasAttr(clang::attr::CUDADevice)) matches the function declaration of
-f. If the matcher is used from clang-query, attr::Kind parameter should be
-passed as a quoted string. e.g., hasAttr("attr::CUDADevice").
+  __attribute__((device)) void f() {}
+
+The matcher decl(hasAttr(clang::attr::CUDADevice))
+matches f.
+If the matcher is used from clang-query, attr::Kind
+parameter should be passed as a quoted string. e.g.,
+hasAttr("attr::CUDADevice").
 </pre></td></tr>
 
 
@@ -4144,6 +5561,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Does not match if only part of the statement is expanded from that macro or
 if different parts of the statement are expanded from different
 appearances of the macro.
+
+Given
+  #define A 0
+  #define B A
+  int c = B;
+
+The matcher integerLiteral(isExpandedFromMacro("A"))
+matches the literal expanded at the initializer B of the variable
+c .
 </pre></td></tr>
 
 
@@ -4151,12 +5577,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching0"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInFileMatching("AST.*"))
-  #include "ASTMatcher.h"
-  class X {};
-ASTMatcher.h:
-  class Y {};
+Given the headers Y.h
+  #pragma once
+  typedef int my_y_int;
+and X.h
+  #pragma once
+  typedef int my_x_int;
+and the source code
+  #include "X.h"
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_x_int b = 1;
+  my_y_int c = 2;
+
+The matcher
+typedefDecl(isExpansionInFileMatching("Y.h"))
+matches typedef int my_y_int,
+but does not match typedef int my_main_file_int or
+typedef int my_x_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 
@@ -4169,12 +5608,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('isExpansionInMainFile0')"><a name="isExpansionInMainFile0Anchor">isExpansionInMainFile</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInMainFile0"><pre>Matches AST nodes that were expanded within the main-file.
 
-Example matches X but not Y
-  (matcher = cxxRecordDecl(isExpansionInMainFile())
-  #include &lt;Y.h&gt;
-  class X {};
-Y.h:
-  class Y {};
+Given the header Y.h
+  #pragma once
+  typedef int my_header_int;
+and the source file
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_header_int b = 1;
+
+The matcher typedefDecl(isExpansionInMainFile())
+matches typedef int my_main_file_int,
+but does not match typedef int my_header_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -4183,12 +5628,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('isExpansionInSystemHeader0')"><a name="isExpansionInSystemHeader0Anchor">isExpansionInSystemHeader</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInSystemHeader0"><pre>Matches AST nodes that were expanded within system-header-files.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInSystemHeader())
+Given the header SystemHeader.h
+  #pragma once
+  int header();
+and the source code
   #include &lt;SystemHeader.h&gt;
-  class X {};
-SystemHeader.h:
-  class Y {};
+  static int main_file();
+
+
+The matcher functionDecl(isExpansionInSystemHeader())
+matches int header(),
+but does not match static int main_file().
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -4197,6 +5647,20 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;</td><td class="name" onclick="toggle('isImplicit0')"><a name="isImplicit0Anchor">isImplicit</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isImplicit0"><pre>Matches an entity that has been implicitly added by the compiler (e.g.
 implicit default/copy constructors).
+
+Given
+  struct S {};
+  void f(S obj) {
+    S copy = obj;
+    [&amp;](){ return copy; };
+  }
+
+
+The matcher cxxConstructorDecl(isImplicit(), isCopyConstructor())
+matches the implicit copy constructor of S.
+The matcher lambdaExpr(forEachLambdaCapture(
+    lambdaCapture(isImplicit()))) matches [&amp;](){ return copy; },
+because it implicitly captures copy .
 </pre></td></tr>
 
 
@@ -4214,11 +5678,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   namespace {
     class vector {}; // #2
     namespace foo {
-      class vector{}; // #3
+      class vector {}; // #3
     }
   }
-cxxRecordDecl(hasName("vector"), isInAnonymousNamespace()) will match
-#1, #2 and #3.
+
+The matcher cxxRecordDecl(hasName("vector"),
+                        isInAnonymousNamespace())
+matches vector,
+twice per declaration at #1, #2 and #3.
 </pre></td></tr>
 
 
@@ -4241,7 +5708,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
       }
     }
   }
-cxxRecordDecl(hasName("vector"), isInStdNamespace()) will match only #1.
+
+The matcher cxxRecordDecl(hasName("vector"), isInStdNamespace())
+matches class vector {} inside of namespace std.
 </pre></td></tr>
 
 
@@ -4251,10 +5720,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   template&lt;typename T&gt; void A(T t) { T i; }
-  A(0);
-  A(0U);
-functionDecl(isInstantiated())
-  matches 'A(int) {...};' and 'A(unsigned) {...}'.
+  void foo() {
+    A(0);
+    A(0U);
+  }
+
+The matcher functionDecl(isInstantiated())
+matches the two instantiations of void A(T t) { T i; } that
+are generated for int , and for int}.
 </pre></td></tr>
 
 
@@ -4262,16 +5735,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isPrivate0"><pre>Matches private C++ declarations and C++ base specifers that specify private
 inheritance.
 
-Examples:
+Given
   class C {
   public:    int a;
   protected: int b;
-  private:   int c; // fieldDecl(isPrivate()) matches 'c'
+  private:   int c;
   };
 
+The matcher fieldDecl(isPrivate())
+matches c.
+
   struct Base {};
-  struct Derived1 : private Base {}; // matches 'Base'
-  class Derived2 : Base {}; // matches 'Base'
+  struct Derived1 : private Base {}; // Base
+  class Derived2 : Base {}; // Base
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPrivate()).bind("base")))
+matches Derived1 and Derived2, with
+cxxBaseSpecifier(isPrivate()) matching
+Base.
 </pre></td></tr>
 
 
@@ -4279,15 +5761,24 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isProtected0"><pre>Matches protected C++ declarations and C++ base specifers that specify
 protected inheritance.
 
-Examples:
+Given
   class C {
   public:    int a;
-  protected: int b; // fieldDecl(isProtected()) matches 'b'
+  protected: int b;
   private:   int c;
   };
 
+The matcher fieldDecl(isProtected())
+matches b.
+
   class Base {};
-  class Derived : protected Base {}; // matches 'Base'
+  class Derived : protected Base {};
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isProtected()).bind("base")))
+matches Derived, with
+cxxBaseSpecifier(isProtected()) matching
+Base.
 </pre></td></tr>
 
 
@@ -4295,16 +5786,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isPublic0"><pre>Matches public C++ declarations and C++ base specifers that specify public
 inheritance.
 
-Examples:
+Given
   class C {
-  public:    int a; // fieldDecl(isPublic()) matches 'a'
+  public:    int a;
   protected: int b;
   private:   int c;
   };
 
+The matcher fieldDecl(isPublic())
+matches a.
+
+Given
   class Base {};
-  class Derived1 : public Base {}; // matches 'Base'
-  struct Derived2 : Base {}; // matches 'Base'
+  class Derived1 : public Base {};
+  struct Derived2 : Base {};
+
+The matcher
+cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPublic()).bind("base")))
+matches Derived1 and Derived2,
+with cxxBaseSpecifier(isPublic()) matching
+public Base and Base.
 </pre></td></tr>
 
 
@@ -4313,20 +5814,24 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 a specific number of designators.
 
 Example: Given
-  point ptarray[10] = { [2].y = 1.0, [0].x = 1.0 };
-  point ptarray2[10] = { [2].y = 1.0, [2].x = 0.0, [0].x = 1.0 };
-designatorCountIs(2)
-  matches '{ [2].y = 1.0, [0].x = 1.0 }',
-  but not '{ [2].y = 1.0, [2].x = 0.0, [0].x = 1.0 }'.
+  struct point2 { double x; double y; };
+  struct point2 ptarray[10] = { [0].x = 1.0 };
+  struct point2 pt = { .x = 2.0 };
+
+The matcher designatedInitExpr(designatorCountIs(2))
+matches [0].x = 1.0, but not .x = 2.0.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1EnumDecl.html">EnumDecl</a>&gt;</td><td class="name" onclick="toggle('isScoped0')"><a name="isScoped0Anchor">isScoped</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isScoped0"><pre>Matches C++11 scoped enum declaration.
 
-Example matches Y (matcher = enumDecl(isScoped()))
+Given
 enum X {};
 enum class Y {};
+
+The matcher enumDecl(isScoped())
+matches enum class Y {}
 </pre></td></tr>
 
 
@@ -4340,8 +5845,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 sizeof is known (std::size_t) and therefore the size of the outer
 sizeof is known.
   template&lt;typename T&gt;
-  void f(T x, T y) { sizeof(sizeof(T() + T()); }
-expr(isInstantiationDependent()) matches sizeof(sizeof(T() + T())
+  void f(T x, T y) { sizeof(T() + T()); }
+
+The matcher expr(isInstantiationDependent())
+matches sizeof(T() + T()),
+(T() + T()),
+T() + T() and T().
 </pre></td></tr>
 
 
@@ -4355,7 +5864,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void add(T x, int y) {
     x + y;
   }
-expr(isTypeDependent()) matches x + y
+
+The matcher expr(isTypeDependent())
+matches x + y and x.
 </pre></td></tr>
 
 
@@ -4366,7 +5877,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 For example, the array bound of "Chars" in the following example is
 value-dependent.
   template&lt;int Size&gt; int f() { return Size; }
-expr(isValueDependent()) matches return Size
+
+The matcher expr(isValueDependent())
+matches the return value Size.
 </pre></td></tr>
 
 
@@ -4374,16 +5887,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="nullPointerConstant0"><pre>Matches expressions that resolve to a null pointer constant, such as
 GNU's __null, C++11's nullptr, or C's NULL macro.
 
-Given:
+Given
+  #define NULL 0
   void *v1 = NULL;
   void *v2 = nullptr;
   void *v3 = __null; // GNU extension
   char *cp = (char *)0;
   int *ip = 0;
   int i = 0;
-expr(nullPointerConstant())
-  matches the initializer for v1, v2, v3, cp, and ip. Does not match the
-  initializer for i.
+
+The matcher expr(nullPointerConstant())
+matches the initializer NULL of v1,
+matches the initializer nullptr of v2,
+matches the initializer __null of v3,
+matches the initializer 0 of cp and
+matches the initializer 0 of ip,
+but does not match the initializer i of i.
 </pre></td></tr>
 
 
@@ -4397,8 +5916,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     int b : 4;
     int c : 2;
   };
-fieldDecl(hasBitWidth(2))
-  matches 'int a;' and 'int c;' but not 'int b;'.
+
+The matcher fieldDecl(hasBitWidth(2))
+matches a and c,
+but not b.
 </pre></td></tr>
 
 
@@ -4410,8 +5931,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     int a : 2;
     int b;
   };
-fieldDecl(isBitField())
-  matches 'int a;' but not 'int b;'.
+
+The matcher fieldDecl(isBitField())
+matches a,
+but does not match b.
 </pre></td></tr>
 
 
@@ -4419,22 +5942,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="equals1"><pre>Matches literals that are equal to the given value of type ValueT.
 
 Given
+void f(char, bool, double, int);
+void foo() {
   f('false, 3.14, 42);
-characterLiteral(equals(0))
-  matches 'cxxBoolLiteral(equals(false)) and cxxBoolLiteral(equals(0))
-  match false
-floatLiteral(equals(3.14)) and floatLiteral(equals(314e-2))
-  match 3.14
-integerLiteral(equals(42))
-  matches 42
+}
+
+The matcher characterLiteral(equals(0U)) matches 'The matchers cxxBoolLiteral(equals(false)) and
+cxxBoolLiteral(equals(0)) match false.
+The matcher floatLiteral(equals(3.14)) matches 3.14.
+The matcher integerLiteral(equals(42)) matches 42.
 
 Note that you cannot directly match a negative numeric literal because the
 minus sign is not part of the literal: It is a unary operator whose operand
 is the positive numeric literal. Instead, you must use a unaryOperator()
 matcher to match the minus sign:
 
-unaryOperator(hasOperatorName("-"),
-              hasUnaryOperand(integerLiteral(equals(13))))
+Given
+  int val = -1;
+
+The matcher unaryOperator(hasOperatorName("-"),
+              hasUnaryOperand(integerLiteral(equals(1))))
+matches -1.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CharacterLiteral.html">CharacterLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBoolLiteralExpr.html">CXXBoolLiteralExpr</a>&gt;,
            Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FloatingLiteral.html">FloatingLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IntegerLiteral.html">IntegerLiteral</a>&gt;
@@ -4452,6 +5980,30 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 "operator" prefix: e.g. "&lt;&lt;".
 
   hasAnyOverloadedOperatorName("+", "-")
+
+Given
+  struct Point { double x; double y; };
+  Point operator+(const Point&amp;, const Point&amp;);
+  Point operator-(const Point&amp;, const Point&amp;);
+
+  Point sub(Point a, Point b) {
+    return b - a;
+  }
+
+
+The matcher functionDecl(hasAnyOverloadedOperatorName("+", "-")),
+which is equivalent to
+functionDecl(anyOf(hasAnyOverloadedOperatorName("+"),
+hasOverloadedOperatorName("-"))),
+matches Point operator+(const Point&amp;, const Point&amp;) and
+Point operator-(const Point&amp;, const Point&amp;).
+The matcher
+cxxOperatorCallExpr(hasAnyOverloadedOperatorName("+", "-")),
+which is equivalent to
+cxxOperatorCallExpr(anyOf(hasOverloadedOperatorName("+"),
+hasOverloadedOperatorName("-"))),
+matches b - a.
+
 Is equivalent to
   anyOf(hasOverloadedOperatorName("+"), hasOverloadedOperatorName("-"))
 </pre></td></tr>
@@ -4460,17 +6012,32 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasDynamicExceptionSpec0')"><a name="hasDynamicExceptionSpec0Anchor">hasDynamicExceptionSpec</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasDynamicExceptionSpec0"><pre>Matches functions that have a dynamic exception specification.
 
-Given:
-  void f();
-  void g() noexcept;
-  void h() noexcept(true);
-  void i() noexcept(false);
-  void j() throw();
-  void k() throw(int);
-  void l() throw(...);
-functionDecl(hasDynamicExceptionSpec()) and
-  functionProtoType(hasDynamicExceptionSpec())
-  match the declarations of j, k, and l, but not f, g, h, or i.
+Given
+  void f(int);
+  void g(int) noexcept;
+  void h(int) noexcept(true);
+  void i(int) noexcept(false);
+  void j(int) throw();
+  void k(int) throw(int);
+  void l(int) throw(...);
+
+The matcher functionDecl(hasDynamicExceptionSpec())
+matches the declarations void j(int) throw(),
+void k(int) throw(int)
+and void l(int) throw(...),
+but does not match void f(int), void g(int) noexcept,
+void h(int) noexcept(true)
+or void i(int) noexcept(true).
+The matcher
+functionProtoType(hasDynamicExceptionSpec()) matches
+the type void (int) throw() of j ,
+the type void (int) throw(int) of k and
+the type void (int) throw(...) of l .
+It does not match
+the type void (int) noexcept of f ,
+the type void (int) noexcept of g ,
+the type void (int) noexcept(int) of h or
+the type void (int) noexcept(...) of i .
 </pre></td></tr>
 
 
@@ -4480,16 +6047,19 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Matches overloaded operator names specified in strings without the
 "operator" prefix: e.g. "&lt;&lt;".
 
-Given:
-  class A { int operator*(); };
+Given
+  struct A { int operator*(); };
   const A &amp;operator&lt;&lt;(const A &amp;a, const A &amp;b);
-  A a;
-  a &lt;&lt; a;   // &lt;-- This matches
+  void f(A a) {
+    a &lt;&lt; a;   // &lt;-- This matches
+  }
+
 
-cxxOperatorCallExpr(hasOverloadedOperatorName("&lt;&lt;"))) matches the
-specified line and
+The matcher cxxOperatorCallExpr(hasOverloadedOperatorName("&lt;&lt;"))
+matches a &lt;&lt; a.
+The matcher
 cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")))
-matches the declaration of A.
+matches struct A { int operator*(); }.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;
 </pre></td></tr>
@@ -4498,9 +6068,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTrailingReturn0')"><a name="hasTrailingReturn0Anchor">hasTrailingReturn</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasTrailingReturn0"><pre>Matches a function declared with a trailing return type.
 
-Example matches Y (matcher = functionDecl(hasTrailingReturn()))
+Given
 int X() {}
 auto Y() -&gt; int {}
+
+The matcher functionDecl(hasTrailingReturn())
+matches auto Y() -&gt; int {}.
 </pre></td></tr>
 
 
@@ -4508,15 +6081,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isConsteval0"><pre>Matches consteval function declarations and if consteval/if ! consteval
 statements.
 
-Given:
+Given
   consteval int a();
   void b() { if consteval {} }
   void c() { if ! consteval {} }
   void d() { if ! consteval {} else {} }
-functionDecl(isConsteval())
-  matches the declaration of "int a()".
-ifStmt(isConsteval())
-  matches the if statement in "void b()", "void c()", "void d()".
+
+The matcher functionDecl(isConsteval())
+matches a.
+The matcher ifStmt(isConsteval())
+matches the if statements
+if consteval {}, if ! consteval {} and
+if ! consteval {} else {}.
 </pre></td></tr>
 
 
@@ -4524,27 +6100,30 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isConstexpr1"><pre>Matches constexpr variable and function declarations,
        and if constexpr.
 
-Given:
+Given
   constexpr int foo = 42;
   constexpr int bar();
   void baz() { if constexpr(1 &gt; 0) {} }
-varDecl(isConstexpr())
-  matches the declaration of foo.
-functionDecl(isConstexpr())
-  matches the declaration of bar.
-ifStmt(isConstexpr())
-  matches the if statement in baz.
+
+The matcher varDecl(isConstexpr())
+matches foo.
+The matcher functionDecl(isConstexpr())
+matches bar.
+The matcher ifStmt(isConstexpr())
+matches if constexpr(1 &gt; 0) {}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isDefaulted0')"><a name="isDefaulted0Anchor">isDefaulted</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isDefaulted0"><pre>Matches defaulted function declarations.
 
-Given:
+Given
   class A { ~A(); };
   class B { ~B() = default; };
-functionDecl(isDefaulted())
-  matches the declaration of ~B, but not ~A.
+
+The matcher functionDecl(isDefaulted())
+  matches ~B() = default,
+but does not match ~A().
 </pre></td></tr>
 
 
@@ -4558,6 +6137,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   extern int vb;  // Doesn't match, as it doesn't define the variable.
   void fa() {}
   void fb();  // Doesn't match, as it has no body.
+
+The matcher tagDecl(isDefinition())
+matches A
+The matcher varDecl(isDefinition())
+matches va
+The matcher functionDecl(isDefinition())
+matches fa
+
   @interface X
   - (void)ma; // Doesn't match, interface is declaration.
   @end
@@ -4565,6 +6152,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   - (void)ma {}
   @end
 
+The matcher objcMethodDecl(isDefinition())
+matches - (void)ma {}
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;
 </pre></td></tr>
@@ -4573,11 +6163,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isDeleted0')"><a name="isDeleted0Anchor">isDeleted</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isDeleted0"><pre>Matches deleted function declarations.
 
-Given:
+Given
   void Func();
   void DeletedFunc() = delete;
-functionDecl(isDeleted())
-  matches the declaration of DeletedFunc, but not Func.
+
+The matcher functionDecl(isDeleted())
+matches DeletedFunc,
+but does not match Func.
 </pre></td></tr>
 
 
@@ -4588,8 +6180,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; void A(T t) { }
   template&lt;&gt; void A(int N) { }
-functionDecl(isExplicitTemplateSpecialization())
-  matches the specialization A&lt;int&gt;().
+
+The matcher functionDecl(isExplicitTemplateSpecialization())
+  matches the specialization template&lt;&gt; void A(int N) { }.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
 </pre></td></tr>
@@ -4598,17 +6191,21 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isExternC0')"><a name="isExternC0Anchor">isExternC</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExternC0"><pre>Matches extern "C" function or variable declarations.
 
-Given:
+Given
   extern "C" void f() {}
   extern "C" { void g() {} }
   void h() {}
   extern "C" int x = 1;
   extern "C" int y = 2;
   int z = 3;
-functionDecl(isExternC())
-  matches the declaration of f and g, but not the declaration of h.
-varDecl(isExternC())
-  matches the declaration of x and y, but not the declaration of z.
+
+The matcher functionDecl(isExternC())
+matches f
+and g.
+The matcher varDecl(isExternC())
+matches x
+and y,
+but does not match z.
 </pre></td></tr>
 
 
@@ -4623,15 +6220,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   inline namespace m {}
   }
   inline int Foo = 5;
-functionDecl(isInline()) will match ::f().
-namespaceDecl(isInline()) will match n::m.
-varDecl(isInline()) will match Foo;
+
+The matcher functionDecl(isInline()) matches f.
+The matcher namespaceDecl(isInline()) matches m.
+The matcher varDecl(isInline()) matches Foo
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isMain0')"><a name="isMain0Anchor">isMain</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isMain0"><pre>Determines whether the function is "main", which is the entry point
 into an executable program.
+
+Given
+  void f();
+  int main() {}
+
+The matcher functionDecl(isMain()) matches int main() {}.
 </pre></td></tr>
 
 
@@ -4643,23 +6247,38 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   [[noreturn]] void a();
   __attribute__((noreturn)) void b();
   struct c { [[noreturn]] c(); };
-functionDecl(isNoReturn())
-  matches all of those except
-  void nope();
+
+The matcher functionDecl(isNoReturn())
+match a, b
+and c
+but do not match nope
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isNoThrow0')"><a name="isNoThrow0Anchor">isNoThrow</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isNoThrow0"><pre>Matches functions that have a non-throwing exception specification.
 
-Given:
-  void f();
-  void g() noexcept;
-  void h() throw();
-  void i() throw(int);
-  void j() noexcept(false);
-functionDecl(isNoThrow()) and functionProtoType(isNoThrow())
-  match the declarations of g, and h, but not f, i or j.
+Given
+  void f(int);
+  void g(int) noexcept;
+  void h(int) noexcept(false);
+  void i(int) throw();
+  void j(int) throw(int);
+
+The matcher functionDecl(isNoThrow())
+matches the declaration void g(int) noexcept
+and void i(int) throw(),
+but does not match void f(int),
+void h(int) noexcept(false)
+or void j(int) throw(int).
+The matcher
+functionProtoType(isNoThrow())
+matches the type void (int) throw() of i
+and the type void (int) noexcept of g,
+but does not match
+the type void (int) of f ,
+the type void (int) noexcept(false) of h or
+the type void (int) throw(int) of j .
 </pre></td></tr>
 
 
@@ -4667,15 +6286,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isStaticStorageClass0"><pre>Matches variable/function declarations that have "static" storage
 class specifier ("static" keyword) written in the source.
 
-Given:
+Given
   static void f() {}
   static int i = 0;
   extern int j;
   int k;
-functionDecl(isStaticStorageClass())
-  matches the function declaration f.
-varDecl(isStaticStorageClass())
-  matches the variable declaration i.
+The matcher functionDecl(isStaticStorageClass())
+  matches f
+The matcher varDecl(isStaticStorageClass())
+  matches i
 </pre></td></tr>
 
 
@@ -4684,18 +6303,36 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 member variable template instantiations.
 
 Given
-  template &lt;typename T&gt; class X {}; class A {}; X&lt;A&gt; x;
-or
-  template &lt;typename T&gt; class X {}; class A {}; template class X&lt;A&gt;;
-or
-  template &lt;typename T&gt; class X {}; class A {}; extern template class X&lt;A&gt;;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
-  matches the template instantiation of X&lt;A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches class X&lt;class A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches template class X&lt;A&gt;
+  template &lt;typename T&gt; class X {};
+  class A {};
+  extern template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches extern template class X&lt;A&gt;
 
 But given
-  template &lt;typename T&gt;  class X {}; class A {};
-  template &lt;&gt; class X&lt;A&gt; {}; X&lt;A&gt; x;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
+  template &lt;typename T&gt;  class X {};
+  class A {};
+  template &lt;&gt; class X&lt;A&gt; {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
   does not match, as X&lt;A&gt; is an explicit template specialization.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
@@ -4711,17 +6348,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void g(int);
   template &lt;typename... Ts&gt; void h(Ts...);
   void i();
+
+The matcher functionDecl(isVariadic())
+matches void f(...),
+but does not match void g(int),
+template &lt;typename... Ts&gt; void h(Ts...),
+or void i().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('isWeak0')"><a name="isWeak0Anchor">isWeak</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isWeak0"><pre>Matches weak function declarations.
 
-Given:
-  void foo() __attribute__((__weakref__("__foo")));
-  void bar();
-functionDecl(isWeak())
-  matches the weak declaration "foo", but not "bar".
+Given
+  static void f();
+  void g() __attribute__((weak));
+The matcher functionDecl(isWeak())
+  matches the weak declaration
+void g() __attribute__((weak)),
+but does not match static void foo_v1().
 </pre></td></tr>
 
 
@@ -4735,43 +6380,71 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void h(int i, int j);
   void j(int i);
   void k(int x, int y, int z, ...);
-functionDecl(parameterCountIs(2))
-  matches g and h
-functionProtoType(parameterCountIs(2))
-  matches g and h
-functionProtoType(parameterCountIs(3))
-  matches k
+The matcher functionDecl(parameterCountIs(2))
+matches g and h
+The matcher functionProtoType(parameterCountIs(1))
+matches the type void (int) of f and j.
+The matcher functionProtoType(parameterCountIs(3)) matches the
+type void (int, int, int, ...) of k.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionProtoType.html">FunctionProtoType</a>&gt;</td><td class="name" onclick="toggle('hasDynamicExceptionSpec1')"><a name="hasDynamicExceptionSpec1Anchor">hasDynamicExceptionSpec</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasDynamicExceptionSpec1"><pre>Matches functions that have a dynamic exception specification.
 
-Given:
-  void f();
-  void g() noexcept;
-  void h() noexcept(true);
-  void i() noexcept(false);
-  void j() throw();
-  void k() throw(int);
-  void l() throw(...);
-functionDecl(hasDynamicExceptionSpec()) and
-  functionProtoType(hasDynamicExceptionSpec())
-  match the declarations of j, k, and l, but not f, g, h, or i.
+Given
+  void f(int);
+  void g(int) noexcept;
+  void h(int) noexcept(true);
+  void i(int) noexcept(false);
+  void j(int) throw();
+  void k(int) throw(int);
+  void l(int) throw(...);
+
+The matcher functionDecl(hasDynamicExceptionSpec())
+matches the declarations void j(int) throw(),
+void k(int) throw(int)
+and void l(int) throw(...),
+but does not match void f(int), void g(int) noexcept,
+void h(int) noexcept(true)
+or void i(int) noexcept(true).
+The matcher
+functionProtoType(hasDynamicExceptionSpec()) matches
+the type void (int) throw() of j ,
+the type void (int) throw(int) of k and
+the type void (int) throw(...) of l .
+It does not match
+the type void (int) noexcept of f ,
+the type void (int) noexcept of g ,
+the type void (int) noexcept(int) of h or
+the type void (int) noexcept(...) of i .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionProtoType.html">FunctionProtoType</a>&gt;</td><td class="name" onclick="toggle('isNoThrow1')"><a name="isNoThrow1Anchor">isNoThrow</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isNoThrow1"><pre>Matches functions that have a non-throwing exception specification.
 
-Given:
-  void f();
-  void g() noexcept;
-  void h() throw();
-  void i() throw(int);
-  void j() noexcept(false);
-functionDecl(isNoThrow()) and functionProtoType(isNoThrow())
-  match the declarations of g, and h, but not f, i or j.
+Given
+  void f(int);
+  void g(int) noexcept;
+  void h(int) noexcept(false);
+  void i(int) throw();
+  void j(int) throw(int);
+
+The matcher functionDecl(isNoThrow())
+matches the declaration void g(int) noexcept
+and void i(int) throw(),
+but does not match void f(int),
+void h(int) noexcept(false)
+or void j(int) throw(int).
+The matcher
+functionProtoType(isNoThrow())
+matches the type void (int) throw() of i
+and the type void (int) noexcept of g,
+but does not match
+the type void (int) of f ,
+the type void (int) noexcept(false) of h or
+the type void (int) throw(int) of j .
 </pre></td></tr>
 
 
@@ -4785,12 +6458,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void h(int i, int j);
   void j(int i);
   void k(int x, int y, int z, ...);
-functionDecl(parameterCountIs(2))
-  matches g and h
-functionProtoType(parameterCountIs(2))
-  matches g and h
-functionProtoType(parameterCountIs(3))
-  matches k
+The matcher functionDecl(parameterCountIs(2))
+matches g and h
+The matcher functionProtoType(parameterCountIs(1))
+matches the type void (int) of f and j.
+The matcher functionProtoType(parameterCountIs(3)) matches the
+type void (int, int, int, ...) of k.
 </pre></td></tr>
 
 
@@ -4798,15 +6471,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isConsteval1"><pre>Matches consteval function declarations and if consteval/if ! consteval
 statements.
 
-Given:
+Given
   consteval int a();
   void b() { if consteval {} }
   void c() { if ! consteval {} }
   void d() { if ! consteval {} else {} }
-functionDecl(isConsteval())
-  matches the declaration of "int a()".
-ifStmt(isConsteval())
-  matches the if statement in "void b()", "void c()", "void d()".
+
+The matcher functionDecl(isConsteval())
+matches a.
+The matcher ifStmt(isConsteval())
+matches the if statements
+if consteval {}, if ! consteval {} and
+if ! consteval {} else {}.
 </pre></td></tr>
 
 
@@ -4814,16 +6490,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isConstexpr2"><pre>Matches constexpr variable and function declarations,
        and if constexpr.
 
-Given:
+Given
   constexpr int foo = 42;
   constexpr int bar();
   void baz() { if constexpr(1 &gt; 0) {} }
-varDecl(isConstexpr())
-  matches the declaration of foo.
-functionDecl(isConstexpr())
-  matches the declaration of bar.
-ifStmt(isConstexpr())
-  matches the if statement in baz.
+
+The matcher varDecl(isConstexpr())
+matches foo.
+The matcher functionDecl(isConstexpr())
+matches bar.
+The matcher ifStmt(isConstexpr())
+matches if constexpr(1 &gt; 0) {}.
 </pre></td></tr>
 
 
@@ -4835,22 +6512,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="equals0"><pre>Matches literals that are equal to the given value of type ValueT.
 
 Given
+void f(char, bool, double, int);
+void foo() {
   f('false, 3.14, 42);
-characterLiteral(equals(0))
-  matches 'cxxBoolLiteral(equals(false)) and cxxBoolLiteral(equals(0))
-  match false
-floatLiteral(equals(3.14)) and floatLiteral(equals(314e-2))
-  match 3.14
-integerLiteral(equals(42))
-  matches 42
+}
+
+The matcher characterLiteral(equals(0U)) matches 'The matchers cxxBoolLiteral(equals(false)) and
+cxxBoolLiteral(equals(0)) match false.
+The matcher floatLiteral(equals(3.14)) matches 3.14.
+The matcher integerLiteral(equals(42)) matches 42.
 
 Note that you cannot directly match a negative numeric literal because the
 minus sign is not part of the literal: It is a unary operator whose operand
 is the positive numeric literal. Instead, you must use a unaryOperator()
 matcher to match the minus sign:
 
-unaryOperator(hasOperatorName("-"),
-              hasUnaryOperand(integerLiteral(equals(13))))
+Given
+  int val = -1;
+
+The matcher unaryOperator(hasOperatorName("-"),
+              hasUnaryOperand(integerLiteral(equals(1))))
+matches -1.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CharacterLiteral.html">CharacterLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBoolLiteralExpr.html">CXXBoolLiteralExpr</a>&gt;,
            Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FloatingLiteral.html">FloatingLiteral</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IntegerLiteral.html">IntegerLiteral</a>&gt;
@@ -4876,14 +6558,30 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
     return l();
   }
 };
+
+The matcher
 lambdaExpr(hasAnyCapture(lambdaCapture(capturesThis())))
-  matches `[this]() { return cc; }`.
+matches [this]() { return cc; }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1LambdaCapture.html">LambdaCapture</a>&gt;</td><td class="name" onclick="toggle('isImplicit2')"><a name="isImplicit2Anchor">isImplicit</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isImplicit2"><pre>Matches an entity that has been implicitly added by the compiler (e.g.
 implicit default/copy constructors).
+
+Given
+  struct S {};
+  void f(S obj) {
+    S copy = obj;
+    [&amp;](){ return copy; };
+  }
+
+
+The matcher cxxConstructorDecl(isImplicit(), isCopyConstructor())
+matches the implicit copy constructor of S.
+The matcher lambdaExpr(forEachLambdaCapture(
+    lambdaCapture(isImplicit()))) matches [&amp;](){ return copy; },
+because it implicitly captures copy .
 </pre></td></tr>
 
 
@@ -4902,14 +6600,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   };
   template &lt;class T&gt;
   class Z {
-    void x() { this-&gt;m; }
+    void x() {
+      this-&gt;m;
+      this-&gt;t;
+      this-&gt;t-&gt;m;
+    }
+    int m;
+    T* t;
   };
-memberExpr(isArrow())
-  matches this-&gt;x, x, y.x, a, this-&gt;b
-cxxDependentScopeMemberExpr(isArrow())
-  matches this-&gt;m
-unresolvedMemberExpr(isArrow())
-  matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
+
+The matcher memberExpr(isArrow())
+matches this-&gt;x, x, a,
+this-&gt;b, this-&gt;m and two times this-&gt;t,
+once for the standalone member expression, and once for the member
+expression that later accesses m .
+Additionally, it does not match this-&gt;t-&gt;t.
+The matcher cxxDependentScopeMemberExpr(isArrow())
+matches this-&gt;t-&gt;m, but not this-&gt;m or this-&gt;t.
+The matcher unresolvedMemberExpr(isArrow())
+matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
 </pre></td></tr>
 
 
@@ -4917,29 +6626,43 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyName0"><pre>Matches NamedDecl nodes that have any of the specified names.
 
 This matcher is only provided as a performance optimization of hasName.
-    hasAnyName(a, b, c)
- is equivalent to, but faster than
-    anyOf(hasName(a), hasName(b), hasName(c))
+
+Given
+  void f(int a, int b);
+
+The matcher namedDecl(hasAnyName("a", "b")),
+which is equivalent to the matcher
+namedDecl(hasAnyName("a", "b")),
+matches int a and int b, but not
+void f(int a, int b).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt;</td><td class="name" onclick="toggle('hasExternalFormalLinkage0')"><a name="hasExternalFormalLinkage0Anchor">hasExternalFormalLinkage</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasExternalFormalLinkage0"><pre>Matches a declaration that has external formal linkage.
 
-Example matches only z (matcher = varDecl(hasExternalFormalLinkage()))
+Given
 void f() {
-  int x;
-  static int y;
+  int a;
+  static int b;
 }
-int z;
+int c;
+static int d;
+The matcher varDecl(hasExternalFormalLinkage())
+matches int c,
+but not int a, static int b or int d.
 
-Example matches f() because it has external formal linkage despite being
-unique to the translation unit as though it has internal likage
-(matcher = functionDecl(hasExternalFormalLinkage()))
+Given
+  namespace {
+    void f() {}
+  }
+  void g() {}
+  static void h() {}
 
-namespace {
-void f() {}
-}
+
+The matcher functionDecl(hasExternalFormalLinkage())
+matches void g() {}, but not void f() {} or
+static void h() {}.
 </pre></td></tr>
 
 
@@ -4950,11 +6673,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 with '&lt;enclosing&gt;::'.
 Does not match typedefs of an underlying type with the given name.
 
-Example matches X (Name == "X")
+Given
   class X;
 
-Example matches X (Name is one of "::a::b::X", "a::b::X", "b::X", "X")
+
+The matcher namedDecl(hasName("X"))
+matches class X.
+
+Given
   namespace a { namespace b { class X; } }
+
+
+The matchers namedDecl(hasName("::a::b::X")),
+namedDecl(hasName("a::b::X")),
+namedDecl(hasName("b::X")) and
+namedDecl(hasName("X"))
+match class X.
 </pre></td></tr>
 
 
@@ -4966,12 +6700,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 prefixing the name with '&lt;enclosing&gt;::'.  Does not match typedefs
 of an underlying type with the given name.
 
-Example matches X (regexp == "::X")
-  class X;
-
-Example matches X (regexp is one of "::X", "^foo::.*X", among others)
+Given
   namespace foo { namespace bar { class X; } }
 
+
+The matcher namedDecl(matchesName("^::foo:.*X"))
+matches class X.
+
 If the matcher is used in clang-query, RegexFlags parameter
 should be passed as a quoted string. e.g: "NoFlags".
 Flags can be combined with '|' example "IgnoreCase | BasicRegex"
@@ -4985,7 +6720,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   namespace n {
   namespace {} // #1
   }
-namespaceDecl(isAnonymous()) will match #1 but not ::n.
+
+The matcher namespaceDecl(isAnonymous())
+matches namespace {}, but not namespace n.
 </pre></td></tr>
 
 
@@ -5000,9 +6737,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   inline namespace m {}
   }
   inline int Foo = 5;
-functionDecl(isInline()) will match ::f().
-namespaceDecl(isInline()) will match n::m.
-varDecl(isInline()) will match Foo;
+
+The matcher functionDecl(isInline()) matches f.
+The matcher namespaceDecl(isInline()) matches m.
+The matcher varDecl(isInline()) matches Foo
 </pre></td></tr>
 
 
@@ -5011,15 +6749,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 specified.
 
 Given
+  void foo() {
+    #pragma omp parallel
+      ;
+    #pragma omp parallel default(none)
+      ;
+    #pragma omp parallel default(shared)
+      ;
+    #pragma omp parallel default(private)
+      ;
+    #pragma omp parallel default(firstprivate)
+      ;
+  }
 
-  #pragma omp parallel
-  #pragma omp parallel default(none)
-  #pragma omp parallel default(shared)
-  #pragma omp parallel default(private)
-  #pragma omp parallel default(firstprivate)
 
-``ompDefaultClause(isFirstPrivateKind())`` matches only
-``default(firstprivate)``.
+The matcher
+ompExecutableDirective(hasAnyClause(ompDefaultClause(isFirstPrivateKind())))
+matches #pragma omp parallel default(firstprivate).
 </pre></td></tr>
 
 
@@ -5027,14 +6773,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isNoneKind0"><pre>Matches if the OpenMP ``default`` clause has ``none`` kind specified.
 
 Given
+  void foo() {
+    #pragma omp parallel
+      ;
+    #pragma omp parallel default(none)
+      ;
+    #pragma omp parallel default(shared)
+      ;
+    #pragma omp parallel default(private)
+      ;
+    #pragma omp parallel default(firstprivate)
+      ;
+  }
 
-  #pragma omp parallel
-  #pragma omp parallel default(none)
-  #pragma omp parallel default(shared)
-  #pragma omp parallel default(private)
-  #pragma omp parallel default(firstprivate)
 
-``ompDefaultClause(isNoneKind())`` matches only ``default(none)``.
+The matcher
+ompExecutableDirective(hasAnyClause(ompDefaultClause(isNoneKind())))
+matches only #pragma omp parallel default(none).
 </pre></td></tr>
 
 
@@ -5043,15 +6798,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 specified.
 
 Given
-
-  #pragma omp parallel
+  void foo() {
+    #pragma omp parallel
+      ;
   #pragma omp parallel default(none)
+      ;
   #pragma omp parallel default(shared)
+      ;
   #pragma omp parallel default(private)
+      ;
   #pragma omp parallel default(firstprivate)
+      ;
+  }
+
 
-``ompDefaultClause(isPrivateKind())`` matches only
-``default(private)``.
+The matcher
+ompExecutableDirective(hasAnyClause(ompDefaultClause(isPrivateKind())))
+matches #pragma omp parallel default(private).
 </pre></td></tr>
 
 
@@ -5059,14 +6822,23 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isSharedKind0"><pre>Matches if the OpenMP ``default`` clause has ``shared`` kind specified.
 
 Given
-
-  #pragma omp parallel
-  #pragma omp parallel default(none)
+  void foo() {
+    #pragma omp parallel
+      ;
+    #pragma omp parallel default(none)
+      ;
   #pragma omp parallel default(shared)
+      ;
   #pragma omp parallel default(private)
+      ;
   #pragma omp parallel default(firstprivate)
+      ;
+  }
 
-``ompDefaultClause(isSharedKind())`` matches only ``default(shared)``.
+
+The matcher
+ompExecutableDirective(hasAnyClause(ompDefaultClause(isSharedKind())))
+matches #pragma omp parallel default(shared).
 </pre></td></tr>
 
 
@@ -5075,13 +6847,21 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 clause kind.
 
 Given
+  void foo() {
+    #pragma omp parallel
+      ;
+    #pragma omp parallel for
+      for (int i = 0; i &lt; 10; ++i) {}
+    #pragma omp          for
+      for (int i = 0; i &lt; 10; ++i) {}
+  }
 
-  #pragma omp parallel
-  #pragma omp parallel for
-  #pragma omp          for
 
-`ompExecutableDirective(isAllowedToContainClause(OMPC_default))`` matches
-``omp parallel`` and ``omp parallel for``.
+The matcher
+ompExecutableDirective(isAllowedToContainClauseKind(
+OpenMPClauseKind::OMPC_default))
+matches #pragma omp parallel
+and #pragma omp parallel for.
 
 If the matcher is use from clang-query, ``OpenMPClauseKind`` parameter
 should be passed as a quoted string. e.g.,
@@ -5094,29 +6874,89 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 i.e., directives that can't have a structured block.
 
 Given
+  void foo() {
+    #pragma omp parallel
+    {
+      #pragma omp taskyield
+    }
+  }
 
-  #pragma omp parallel
-  {}
-  #pragma omp taskyield
 
-``ompExecutableDirective(isStandaloneDirective()))`` matches
-``omp taskyield``.
+The matcher ompExecutableDirective(isStandaloneDirective())
+matches #pragma omp taskyield.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('isDerivedFrom3')"><a name="isDerivedFrom3Anchor">isDerivedFrom</a></td><td>std::string BaseName</td></tr>
 <tr><td colspan="4" class="doc" id="isDerivedFrom3"><pre>Overloaded method as shortcut for isDerivedFrom(hasName(...)).
+
+Matches C++ classes that are directly or indirectly derived from a class
+matching Base, or Objective-C classes that directly or indirectly
+subclass a class matching Base.
+
+Note that a class is not considered to be derived from itself.
+
+Example matches Y, Z, C (Base == hasName("X"))
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+  class Foo {};
+  typedef Foo Alias;
+  class Bar : public Alias {};  // derived from Alias, which is a
+                                // typedef of Foo
+
+
+The matcher cxxRecordDecl(isDerivedFrom("X"))
+matches Y, Z and C.
+The matcher cxxRecordDecl(isDerivedFrom("Foo"))
+matches Bar.
+
+In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
+  @interface NSObject @end
+  @interface Bar : NSObject @end
+
+
+Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('isDirectlyDerivedFrom3')"><a name="isDirectlyDerivedFrom3Anchor">isDirectlyDerivedFrom</a></td><td>std::string BaseName</td></tr>
 <tr><td colspan="4" class="doc" id="isDirectlyDerivedFrom3"><pre>Overloaded method as shortcut for isDirectlyDerivedFrom(hasName(...)).
+
+Given
+  struct Base {};
+  struct DirectlyDerived : public Base {};
+  struct IndirectlyDerived : public DirectlyDerived {};
+
+
+The matcher cxxRecordDecl(isDirectlyDerivedFrom("Base"))
+matches DirectlyDerived, but not
+IndirectlyDerived.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('isSameOrDerivedFrom3')"><a name="isSameOrDerivedFrom3Anchor">isSameOrDerivedFrom</a></td><td>std::string BaseName</td></tr>
-<tr><td colspan="4" class="doc" id="isSameOrDerivedFrom3"><pre>Overloaded method as shortcut for
+<tr><td colspan="4" class="doc" id="isSameOrDerivedFrom3"><pre>Similar to isDerivedFrom(), but also matches classes that directly
+match Base.
+Overloaded method as shortcut for
 isSameOrDerivedFrom(hasName(...)).
+
+Given
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+The matcher
+cxxRecordDecl(isSameOrDerivedFrom("X"), isDefinition())
+matches class X {}, class Y : public X {},
+class Z : public Y {} and class C : public B {}.
 </pre></td></tr>
 
 
@@ -5124,12 +6964,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountAtLeast3"><pre>Checks that a call expression or a constructor call expression has at least
 the specified number of arguments (including absent default arguments).
 
-Example matches f(0, 0) and g(0, 0, 0)
-(matcher = callExpr(argumentCountAtLeast(2)))
+Given
   void f(int x, int y);
   void g(int x, int y, int z);
-  f(0, 0);
-  g(0, 0, 0);
+  void foo() {
+    f(0, 0);
+    g(0, 0, 0);
+  }
+The matcher callExpr(argumentCountAtLeast(2))
+matches f(0, 0) and g(0, 0, 0)
 </pre></td></tr>
 
 
@@ -5137,9 +6980,13 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="argumentCountIs3"><pre>Checks that a call expression or a constructor call expression has
 a specific number of arguments (including absent default arguments).
 
-Example matches f(0, 0) (matcher = callExpr(argumentCountIs(2)))
+Given
   void f(int x, int y);
-  f(0, 0);
+  void foo() {
+    f(0, 0);
+  }
+The matcher callExpr(argumentCountIs(2))
+matches f(0, 0)
 </pre></td></tr>
 
 
@@ -5147,24 +6994,27 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnySelector0"><pre>Matches when at least one of the supplied string equals to the
 Selector.getAsString()
 
- matcher = objCMessageExpr(hasSelector("methodA:", "methodB:"));
- matches both of the expressions below:
     [myObj methodA:argA];
     [myObj methodB:argB];
+
+ The matcher objCMessageExpr(hasSelector("methodA:", "methodB:"));
+ matches [myObj methodA:argA]; and [myObj methodB:argB];
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('hasKeywordSelector0')"><a name="hasKeywordSelector0Anchor">hasKeywordSelector</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasKeywordSelector0"><pre>Matches when the selector is a keyword selector
 
-objCMessageExpr(hasKeywordSelector()) matches the generated setFrame
-message expression in
-
+Given
   UIWebView *webView = ...;
   CGRect bodyFrame = webView.frame;
   bodyFrame.size.height = self.bodyContentHeight;
   webView.frame = bodyFrame;
   //     ^---- matches here
+
+
+The matcher objCMessageExpr(hasKeywordSelector()) matches the
+generated setFrame message expression in
 </pre></td></tr>
 
 
@@ -5179,56 +7029,68 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('hasSelector0')"><a name="hasSelector0Anchor">hasSelector</a></td><td>std::string BaseName</td></tr>
 <tr><td colspan="4" class="doc" id="hasSelector0"><pre>Matches when BaseName == Selector.getAsString()
 
- matcher = objCMessageExpr(hasSelector("loadHTMLString:baseURL:"));
- matches the outer message expr in the code below, but NOT the message
- invocation for self.bodyView.
     [self.bodyView loadHTMLString:html baseURL:NULL];
+
+The matcher
+objCMessageExpr(hasSelector("loadHTMLString:baseURL:")); matches
+the outer message expr in the code below, but NOT the message invocation
+for self.bodyView.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('hasUnarySelector0')"><a name="hasUnarySelector0Anchor">hasUnarySelector</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasUnarySelector0"><pre>Matches when the selector is a Unary Selector
 
- matcher = objCMessageExpr(matchesSelector(hasUnarySelector());
- matches self.bodyView in the code below, but NOT the outer message
- invocation of "loadHTMLString:baseURL:".
+Given
     [self.bodyView loadHTMLString:html baseURL:NULL];
+
+
+ The matcher objCMessageExpr(matchesSelector(hasUnarySelector());
+ matches self.bodyView, but does not match the outer message
+ invocation of "loadHTMLString:baseURL:".
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('isClassMessage0')"><a name="isClassMessage0Anchor">isClassMessage</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isClassMessage0"><pre>Returns true when the Objective-C message is sent to a class.
 
-Example
-matcher = objcMessageExpr(isClassMessage())
-matches
+Given
   [NSString stringWithFormat:@"format"];
-but not
   NSString *x = @"hello";
   [x containsString:@"h"];
+
+The matcher objcMessageExpr(isClassMessage())
+matches [NSString stringWithFormat:@"format"];
+but does not match [[x containsString:@"h"]
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('isInstanceMessage0')"><a name="isInstanceMessage0Anchor">isInstanceMessage</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isInstanceMessage0"><pre>Returns true when the Objective-C message is sent to an instance.
 
-Example
-matcher = objcMessageExpr(isInstanceMessage())
-matches
+Given
   NSString *x = @"hello";
   [x containsString:@"h"];
-but not
   [NSString stringWithFormat:@"format"];
+
+The matcher objcMessageExpr(isInstanceMessage())
+matches [x containsString:@"h"];
+but does not match [NSString stringWithFormat:@"format"];
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('matchesSelector0')"><a name="matchesSelector0Anchor">matchesSelector</a></td><td>StringRef RegExp, Regex::RegexFlags Flags = NoFlags</td></tr>
 <tr><td colspan="4" class="doc" id="matchesSelector0"><pre>Matches ObjC selectors whose name contains
 a substring matched by the given RegExp.
- matcher = objCMessageExpr(matchesSelector("loadHTMLStringmatches the outer message expr in the code below, but NOT the message
- invocation for self.bodyView.
+
+Given
     [self.bodyView loadHTMLString:html baseURL:NULL];
 
+
+The matcher
+objCMessageExpr(matchesSelector("loadHTMLStringmatches the outer message expr in the code below, but NOT the message
+invocation for self.bodyView.
+
 If the matcher is used in clang-query, RegexFlags parameter
 should be passed as a quoted string. e.g: "NoFlags".
 Flags can be combined with '|' example "IgnoreCase | BasicRegex"
@@ -5238,25 +7100,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('numSelectorArgs0')"><a name="numSelectorArgs0Anchor">numSelectorArgs</a></td><td>unsigned N</td></tr>
 <tr><td colspan="4" class="doc" id="numSelectorArgs0"><pre>Matches when the selector has the specified number of arguments
 
- matcher = objCMessageExpr(numSelectorArgs(0));
- matches self.bodyView in the code below
-
- matcher = objCMessageExpr(numSelectorArgs(2));
- matches the invocation of "loadHTMLString:baseURL:" but not that
- of self.bodyView
     [self.bodyView loadHTMLString:html baseURL:NULL];
+
+The matcher objCMessageExpr(numSelectorArgs(0))
+matches self.bodyView.
+The matcher objCMessageExpr(numSelectorArgs(2))
+matches the invocation of loadHTMLString:baseURL:
+but does not match self.bodyView
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isClassMethod0')"><a name="isClassMethod0Anchor">isClassMethod</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isClassMethod0"><pre>Returns true when the Objective-C method declaration is a class method.
 
-Example
-matcher = objcMethodDecl(isClassMethod())
-matches
+Given
 @interface I + (void)foo; @end
-but not
 @interface I - (void)bar; @end
+
+The matcher objcMethodDecl(isClassMethod())
+matches @interface I + (void)foo; @end
+but does not match interface I + (void)foo; @end
 </pre></td></tr>
 
 
@@ -5270,6 +7133,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   extern int vb;  // Doesn't match, as it doesn't define the variable.
   void fa() {}
   void fb();  // Doesn't match, as it has no body.
+
+The matcher tagDecl(isDefinition())
+matches A
+The matcher varDecl(isDefinition())
+matches va
+The matcher functionDecl(isDefinition())
+matches fa
+
   @interface X
   - (void)ma; // Doesn't match, interface is declaration.
   @end
@@ -5277,6 +7148,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   - (void)ma {}
   @end
 
+The matcher objcMethodDecl(isDefinition())
+matches - (void)ma {}
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;
 </pre></td></tr>
@@ -5285,33 +7159,39 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;</td><td class="name" onclick="toggle('isInstanceMethod0')"><a name="isInstanceMethod0Anchor">isInstanceMethod</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isInstanceMethod0"><pre>Returns true when the Objective-C method declaration is an instance method.
 
-Example
-matcher = objcMethodDecl(isInstanceMethod())
-matches
+Given
 @interface I - (void)bar; @end
-but not
 @interface I + (void)foo; @end
+
+The matcher objcMethodDecl(isInstanceMethod())
+matches @interface I - (void)bar; @end
+but does not match @interface I - (void)foo; @end
+
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ParmVarDecl.html">ParmVarDecl</a>&gt;</td><td class="name" onclick="toggle('hasDefaultArgument0')"><a name="hasDefaultArgument0Anchor">hasDefaultArgument</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasDefaultArgument0"><pre>Matches a declaration that has default arguments.
 
-Example matches y (matcher = parmVarDecl(hasDefaultArgument()))
-void x(int val) {}
-void y(int val = 0) {}
+Given
+  void x(int val) {}
+  void y(int val = 0) {}
+
+
+The matcher parmVarDecl(hasDefaultArgument())
+matches int val = 0.
 
 Deprecated. Use hasInitializer() instead to be able to
 match on the contents of the default argument.  For example:
 
-void x(int val = 7) {}
-void y(int val = 42) {}
-parmVarDecl(hasInitializer(integerLiteral(equals(42))))
-  matches the parameter of y
+Given
+  void x(int val = 7) {}
+  void y(int val = 42) {}
+
 
-A matcher such as
-  parmVarDecl(hasInitializer(anything()))
-is equivalent to parmVarDecl(hasDefaultArgument()).
+The matcher
+parmVarDecl(hasInitializer(integerLiteral(equals(42)))),
+matches int val = 42.
 </pre></td></tr>
 
 
@@ -5326,9 +7206,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 void f(int a, int b, int c) {
 }
 
-``parmVarDecl(isAtPosition(0))`` matches ``int a``.
-
-``parmVarDecl(isAtPosition(1))`` matches ``int b``.
+The matcher parmVarDecl(isAtPosition(0)) matches
+a. The matcher parmVarDecl(isAtPosition(1))
+matches b.
 </pre></td></tr>
 
 
@@ -5338,8 +7218,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   class Y { public: void x(); };
   void z() { Y* y; y-&gt;x(); }
-cxxMemberCallExpr(on(hasType(asString("class Y *"))))
-  matches y-&gt;x()
+
+The matcher cxxMemberCallExpr(on(hasType(asString("Y *"))))
+matches y-&gt;x()
 </pre></td></tr>
 
 
@@ -5350,10 +7231,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   class X { int a; int b; };
-cxxRecordDecl(
+
+The matcher cxxRecordDecl(
     has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))
-  matches the class X, as a and b have the same type.
+  matches X, as a and b have the same type.
 
 Note that when multiple matches are involved via forEach* matchers,
 equalsBoundNodes acts as a filter.
@@ -5372,12 +7254,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   typedef const int const_int;
-  const_int i;
-  int *const j;
+  const_int i = 0;
+  int *const j = nullptr;
   int *volatile k;
   int m;
-varDecl(hasType(hasLocalQualifiers())) matches only j and k.
-i is const-qualified but the qualifier is not local.
+
+
+The matcher varDecl(hasType(hasLocalQualifiers())) matches only
+j and k. is
+const-qualified but the qualifier is not local.
 </pre></td></tr>
 
 
@@ -5388,9 +7273,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void a(char);
   void b(wchar_t);
   void c(double);
+
+
+The matcher
 functionDecl(hasAnyParameter(hasType(isAnyCharacter())))
-matches "a(char)", "b(wchar_t)", but not "c(double)".
-</pre></td></tr>
+a, b, but not </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('isAnyPointer0')"><a name="isAnyPointer0Anchor">isAnyPointer</a></td><td></td></tr>
@@ -5406,8 +7293,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   Foo *f;
 
   int j;
-varDecl(hasType(isAnyPointer()))
-  matches "int *i" and "Foo *f", but not "int j".
+
+The matcher varDecl(hasType(isAnyPointer()))
+int *i and Foo *f, but not int j.
 </pre></td></tr>
 
 
@@ -5421,9 +7309,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void c(const int);
   void d(const int*);
   void e(int const) {};
+The matcher
 functionDecl(hasAnyParameter(hasType(isConstQualified())))
-  matches "void b(int const)", "void c(const int)" and
-  "void e(int const) {}". It does not match d as there
+  matches b, c and
+  e.
+  It does not match as there
   is no top-level const on the parameter type "const int *".
 </pre></td></tr>
 
@@ -5435,8 +7325,8 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void a(int);
   void b(long);
   void c(double);
-functionDecl(hasAnyParameter(hasType(isInteger())))
-matches "a(int)", "b(long)", but not "c(double)".
+The matcher functionDecl(hasAnyParameter(hasType(isInteger())))
+a, b, but not c.
 </pre></td></tr>
 
 
@@ -5447,8 +7337,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void a(int);
   void b(unsigned long);
   void c(double);
-functionDecl(hasAnyParameter(hasType(isSignedInteger())))
-matches "a(int)", but not "b(unsigned long)" and "c(double)".
+The matcher
+functionDecl(hasAnyParameter(hasType(isSignedInteger()))) matches
+a, but not and not
 </pre></td></tr>
 
 
@@ -5459,8 +7350,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void a(int);
   void b(unsigned long);
   void c(double);
+The matcher
 functionDecl(hasAnyParameter(hasType(isUnsignedInteger())))
-matches "b(unsigned long)", but not "a(int)" and "c(double)".
+matches b,
+but does not match a and c.
 </pre></td></tr>
 
 
@@ -5474,9 +7367,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   void c(volatile int);
   void d(volatile int*);
   void e(int volatile) {};
+The matcher
 functionDecl(hasAnyParameter(hasType(isVolatileQualified())))
-  matches "void b(int volatile)", "void c(volatile int)" and
-  "void e(int volatile) {}". It does not match d as there
+  matches b, c and
+  e.
+  It does not match as there
   is no top-level volatile on the parameter type "volatile int *".
 </pre></td></tr>
 
@@ -5488,10 +7383,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   class X { int a; int b; };
-cxxRecordDecl(
+
+The matcher cxxRecordDecl(
     has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))
-  matches the class X, as a and b have the same type.
+  matches X, as a and b have the same type.
 
 Note that when multiple matches are involved via forEach* matchers,
 equalsBoundNodes acts as a filter.
@@ -5504,7 +7400,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('equalsNode1')"><a name="equalsNode1Anchor">equalsNode</a></td><td>const Stmt* Other</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('equalsNode1')"><a name="equalsNode1Anchor">equalsNode</a></td><td>const Stmt * Other</td></tr>
 <tr><td colspan="4" class="doc" id="equalsNode1"><pre>Matches if a node equals another node.
 
 Stmt has pointer identity in the AST.
@@ -5516,6 +7412,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Does not match if only part of the statement is expanded from that macro or
 if different parts of the statement are expanded from different
 appearances of the macro.
+
+Given
+  #define A 0
+  #define B A
+  int c = B;
+
+The matcher integerLiteral(isExpandedFromMacro("A"))
+matches the literal expanded at the initializer B of the variable
+c .
 </pre></td></tr>
 
 
@@ -5523,12 +7428,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching1"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInFileMatching("AST.*"))
-  #include "ASTMatcher.h"
-  class X {};
-ASTMatcher.h:
-  class Y {};
+Given the headers Y.h
+  #pragma once
+  typedef int my_y_int;
+and X.h
+  #pragma once
+  typedef int my_x_int;
+and the source code
+  #include "X.h"
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_x_int b = 1;
+  my_y_int c = 2;
+
+The matcher
+typedefDecl(isExpansionInFileMatching("Y.h"))
+matches typedef int my_y_int,
+but does not match typedef int my_main_file_int or
+typedef int my_x_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 
@@ -5541,12 +7459,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('isExpansionInMainFile1')"><a name="isExpansionInMainFile1Anchor">isExpansionInMainFile</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInMainFile1"><pre>Matches AST nodes that were expanded within the main-file.
 
-Example matches X but not Y
-  (matcher = cxxRecordDecl(isExpansionInMainFile())
-  #include &lt;Y.h&gt;
-  class X {};
-Y.h:
-  class Y {};
+Given the header Y.h
+  #pragma once
+  typedef int my_header_int;
+and the source file
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_header_int b = 1;
+
+The matcher typedefDecl(isExpansionInMainFile())
+matches typedef int my_main_file_int,
+but does not match typedef int my_header_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -5555,12 +7479,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('isExpansionInSystemHeader1')"><a name="isExpansionInSystemHeader1Anchor">isExpansionInSystemHeader</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInSystemHeader1"><pre>Matches AST nodes that were expanded within system-header-files.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInSystemHeader())
+Given the header SystemHeader.h
+  #pragma once
+  int header();
+and the source code
   #include &lt;SystemHeader.h&gt;
-  class X {};
-SystemHeader.h:
-  class Y {};
+  static int main_file();
+
+
+The matcher functionDecl(isExpansionInSystemHeader())
+matches int header(),
+but does not match static int main_file().
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -5571,14 +7500,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   int j;
-  template&lt;typename T&gt; void A(T t) { T i; j += 42;}
-  A(0);
-  A(0U);
-declStmt(isInTemplateInstantiation())
-  matches 'int i;' and 'unsigned i'.
-unless(stmt(isInTemplateInstantiation()))
-  will NOT match j += 42; as it's shared between the template definition and
-  instantiation.
+  template&lt;typename T&gt; void A(T t) { T i; }
+  void foo() {
+    A(0);
+    A(0U);
+  }
+
+The matcher declStmt(isInTemplateInstantiation())
+matches T i; twice, once for int and once for
+int}.
+The matcher declStmt(unless(isInTemplateInstantiation())) will
+match T i; once inside the template definition, but not for any of
+the instantiated bodies.
 </pre></td></tr>
 
 
@@ -5592,21 +7525,28 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   char *s = "abcd";
   wchar_t *ws = L"abcd";
   char *w = "a";
-constantArrayType(hasSize(42))
-  matches "int a[42]" and "int b[2 * 21]"
-stringLiteral(hasSize(4))
-  matches "abcd", L"abcd"
+
+The matcher constantArrayType(hasSize(42))
+matches int[42] twice.
+The matcher stringLiteral(hasSize(4))
+matches "abcd" and L"abcd".
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;</td><td class="name" onclick="toggle('isClass0')"><a name="isClass0Anchor">isClass</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isClass0"><pre>Matches TagDecl object that are spelled with "class."
 
-Example matches C, but not S, U or E.
+Given
   struct S {};
   class C {};
   union U {};
-  enum E {};
+  enum E { Ok };
+
+The matcher tagDecl(isClass())
+matches class C,
+but does not match struct S,
+union U
+or enum E.
 </pre></td></tr>
 
 
@@ -5620,6 +7560,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   extern int vb;  // Doesn't match, as it doesn't define the variable.
   void fa() {}
   void fb();  // Doesn't match, as it has no body.
+
+The matcher tagDecl(isDefinition())
+matches A
+The matcher varDecl(isDefinition())
+matches va
+The matcher functionDecl(isDefinition())
+matches fa
+
   @interface X
   - (void)ma; // Doesn't match, interface is declaration.
   @end
@@ -5627,6 +7575,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   - (void)ma {}
   @end
 
+The matcher objcMethodDecl(isDefinition())
+matches - (void)ma {}
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;
 </pre></td></tr>
@@ -5635,11 +7586,16 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;</td><td class="name" onclick="toggle('isEnum0')"><a name="isEnum0Anchor">isEnum</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isEnum0"><pre>Matches TagDecl object that are spelled with "enum."
 
-Example matches E, but not C, S or U.
+Given
   struct S {};
   class C {};
   union U {};
-  enum E {};
+  enum E { Ok };
+
+The matcher tagDecl(isEnum())
+matches enum E { Ok },
+but does not match struct S {},
+class C {} or union U {}.
 </pre></td></tr>
 
 
@@ -5650,18 +7606,30 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   struct S {};
   class C {};
   union U {};
-  enum E {};
+  enum E { Ok };
+
+The matcher tagDecl(isStruct())
+matches struct S,
+but does not match class C,
+union U
+or enum E.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;</td><td class="name" onclick="toggle('isUnion0')"><a name="isUnion0Anchor">isUnion</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isUnion0"><pre>Matches TagDecl object that are spelled with "union."
 
-Example matches U, but not C, S or E.
+Given
   struct S {};
   class C {};
   union U {};
-  enum E {};
+  enum E { Ok };
+
+The matcher tagDecl(isUnion())
+matches union U,
+does not match struct S,
+class C
+or enum E.
 </pre></td></tr>
 
 
@@ -5675,9 +7643,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;int T&gt; struct C {};
   C&lt;42&gt; c;
-classTemplateSpecializationDecl(
+
+The matcher classTemplateSpecializationDecl(
   hasAnyTemplateArgument(equalsIntegralValue("42")))
-  matches the implicit instantiation of C in C&lt;42&gt;.
+matches the implicitly declared specialization
+struct C&lt;42&gt; from the instantiation for the type of the
+variable c .
 </pre></td></tr>
 
 
@@ -5687,10 +7658,12 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;int T&gt; struct C {};
   C&lt;42&gt; c;
-classTemplateSpecializationDecl(
+
+The matcher classTemplateSpecializationDecl(
   hasAnyTemplateArgument(isIntegral()))
-  matches the implicit instantiation of C in C&lt;42&gt;
-  with isIntegral() matching 42.
+matches the implicitly declared specialization
+struct C&lt;42&gt; from the instantiation for the type of the
+variable c .
 </pre></td></tr>
 
 
@@ -5700,8 +7673,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; struct C {};
   C&lt;int&gt; c;
+
+The matcher
 classTemplateSpecializationDecl(templateArgumentCountIs(1))
-  matches C&lt;int&gt;.
+matches struct C&lt;int&gt;.
 </pre></td></tr>
 
 
@@ -5710,6 +7685,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Does not match if only part of the statement is expanded from that macro or
 if different parts of the statement are expanded from different
 appearances of the macro.
+
+Given
+  #define A 0
+  #define B A
+  int c = B;
+
+The matcher integerLiteral(isExpandedFromMacro("A"))
+matches the literal expanded at the initializer B of the variable
+c .
 </pre></td></tr>
 
 
@@ -5717,12 +7701,25 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isExpansionInFileMatching2"><pre>Matches AST nodes that were expanded within files whose name is
 partially matching a given regex.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInFileMatching("AST.*"))
-  #include "ASTMatcher.h"
-  class X {};
-ASTMatcher.h:
-  class Y {};
+Given the headers Y.h
+  #pragma once
+  typedef int my_y_int;
+and X.h
+  #pragma once
+  typedef int my_x_int;
+and the source code
+  #include "X.h"
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_x_int b = 1;
+  my_y_int c = 2;
+
+The matcher
+typedefDecl(isExpansionInFileMatching("Y.h"))
+matches typedef int my_y_int,
+but does not match typedef int my_main_file_int or
+typedef int my_x_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 
@@ -5735,12 +7732,18 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('isExpansionInMainFile2')"><a name="isExpansionInMainFile2Anchor">isExpansionInMainFile</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInMainFile2"><pre>Matches AST nodes that were expanded within the main-file.
 
-Example matches X but not Y
-  (matcher = cxxRecordDecl(isExpansionInMainFile())
-  #include &lt;Y.h&gt;
-  class X {};
-Y.h:
-  class Y {};
+Given the header Y.h
+  #pragma once
+  typedef int my_header_int;
+and the source file
+  #include "Y.h"
+  typedef int my_main_file_int;
+  my_main_file_int a = 0;
+  my_header_int b = 1;
+
+The matcher typedefDecl(isExpansionInMainFile())
+matches typedef int my_main_file_int,
+but does not match typedef int my_header_int.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -5749,12 +7752,17 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('isExpansionInSystemHeader2')"><a name="isExpansionInSystemHeader2Anchor">isExpansionInSystemHeader</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExpansionInSystemHeader2"><pre>Matches AST nodes that were expanded within system-header-files.
 
-Example matches Y but not X
-    (matcher = cxxRecordDecl(isExpansionInSystemHeader())
+Given the header SystemHeader.h
+  #pragma once
+  int header();
+and the source code
   #include &lt;SystemHeader.h&gt;
-  class X {};
-SystemHeader.h:
-  class Y {};
+  static int main_file();
+
+
+The matcher functionDecl(isExpansionInSystemHeader())
+matches int header(),
+but does not match static int main_file().
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;
 </pre></td></tr>
@@ -5765,8 +7773,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
  struct S { bool func(); };
-functionDecl(returns(booleanType()))
-  matches "bool func();"
+
+The matcher functionDecl(returns(booleanType()))
+func
 </pre></td></tr>
 
 
@@ -5777,10 +7786,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   class X { int a; int b; };
-cxxRecordDecl(
+
+The matcher cxxRecordDecl(
     has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))
-  matches the class X, as a and b have the same type.
+  matches X, as a and b have the same type.
 
 Note that when multiple matches are involved via forEach* matchers,
 equalsBoundNodes acts as a filter.
@@ -5793,7 +7803,7 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('equalsNode2')"><a name="equalsNode2Anchor">equalsNode</a></td><td>const Type* Other</td></tr>
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td><td class="name" onclick="toggle('equalsNode2')"><a name="equalsNode2Anchor">equalsNode</a></td><td>const Type * Other</td></tr>
 <tr><td colspan="4" class="doc" id="equalsNode2"><pre>Matches if a node equals another node.
 
 Type has pointer identity in the AST.
@@ -5806,8 +7816,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   int i;
   float f;
-realFloatingPointType()
-  matches "float f" but not "int i"
+The matcher type(realFloatingPointType())
+matches float
+but does not match int.
 </pre></td></tr>
 
 
@@ -5816,8 +7827,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
  struct S { void func(); };
-functionDecl(returns(voidType()))
-  matches "void func();"
+
+
+The matcher functionDecl(returns(voidType()))
+func
 </pre></td></tr>
 
 
@@ -5826,9 +7839,10 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 
 Given
   int x;
-  int s = sizeof(x) + alignof(x)
-unaryExprOrTypeTraitExpr(ofKind(UETT_SizeOf))
-  matches sizeof(x)
+  int s = sizeof(x) + alignof(x);
+
+The matcher unaryExprOrTypeTraitExpr(ofKind(UETT_SizeOf))
+matches sizeof(x)
 
 If the matcher is use from clang-query, UnaryExprOrTypeTrait parameter
 should be passed as a quoted string. e.g., ofKind("UETT_SizeOf").
@@ -5839,9 +7853,26 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyOperatorName3"><pre>Matches operator expressions (binary or unary) that have any of the
 specified names.
 
+It provides a compact way of writing if an operator has any of the specified
+names:
+The matcher
    hasAnyOperatorName("+", "-")
- Is equivalent to
-   anyOf(hasOperatorName("+"), hasOperatorName("-"))
+Is equivalent to
+   hasOperatorName("-"))}
+
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+void bar(bool a, bool b) {
+  a &amp;&amp; b;
+ }
+
+The matcher binaryOperator(hasAnyOperatorName("||", "&amp;&amp;"))
+matches a || b and a &amp;&amp; b.
+The matcher unaryOperator(hasAnyOperatorName("-", "!"))
+matches !(a || b).
 </pre></td></tr>
 
 
@@ -5849,15 +7880,22 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperatorName4"><pre>Matches the operator Name of operator expressions and fold expressions
 (binary or unary).
 
-Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
-  !(a || b)
+Given
+void foo(bool a, bool b) {
+  !(a || b);
+ }
+
+The matcher binaryOperator(hasOperatorName("||"))
+matches a || b
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasOperatorName("+")))
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
   }
+
+The matcher cxxFoldExpr(hasOperatorName("+"))
+ matches (0 + ... + args).
 </pre></td></tr>
 
 
@@ -5876,40 +7914,57 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   };
   template &lt;class T&gt;
   class Z {
-    void x() { this-&gt;m; }
+    void x() {
+      this-&gt;m;
+      this-&gt;t;
+      this-&gt;t-&gt;m;
+    }
+    int m;
+    T* t;
   };
-memberExpr(isArrow())
-  matches this-&gt;x, x, y.x, a, this-&gt;b
-cxxDependentScopeMemberExpr(isArrow())
-  matches this-&gt;m
-unresolvedMemberExpr(isArrow())
-  matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
+
+The matcher memberExpr(isArrow())
+matches this-&gt;x, x, a,
+this-&gt;b, this-&gt;m and two times this-&gt;t,
+once for the standalone member expression, and once for the member
+expression that later accesses m .
+Additionally, it does not match this-&gt;t-&gt;t.
+The matcher cxxDependentScopeMemberExpr(isArrow())
+matches this-&gt;t-&gt;m, but not this-&gt;m or this-&gt;t.
+The matcher unresolvedMemberExpr(isArrow())
+matches this-&gt;f&lt;T&gt;, f&lt;T&gt;
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('hasAutomaticStorageDuration0')"><a name="hasAutomaticStorageDuration0Anchor">hasAutomaticStorageDuration</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasAutomaticStorageDuration0"><pre>Matches a variable declaration that has automatic storage duration.
 
-Example matches x, but not y, z, or a.
-(matcher = varDecl(hasAutomaticStorageDuration())
+Given
 void f() {
   int x;
   static int y;
   thread_local int z;
 }
 int a;
+
+The matcher varDecl(hasAutomaticStorageDuration())
+matches x
+but does not match y, z or
+a
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('hasGlobalStorage0')"><a name="hasGlobalStorage0Anchor">hasGlobalStorage</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasGlobalStorage0"><pre>Matches a variable declaration that does not have local storage.
 
-Example matches y and z (matcher = varDecl(hasGlobalStorage())
+Given
 void f() {
   int x;
   static int y;
 }
 int z;
+The matcher varDecl(hasGlobalStorage())
+matches y and z
 </pre></td></tr>
 
 
@@ -5917,12 +7972,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasLocalStorage0"><pre>Matches a variable declaration that has function scope and is a
 non-static local variable.
 
-Example matches x (matcher = varDecl(hasLocalStorage())
+Given
 void f() {
   int x;
   static int y;
 }
 int z;
+The matcher varDecl(hasLocalStorage())
+matches x
 </pre></td></tr>
 
 
@@ -5939,22 +7996,28 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 int a;
 static int b;
 extern int c;
-varDecl(hasStaticStorageDuration())
-  matches the function declaration y, a, b and c.
+
+The matcher varDecl(hasStaticStorageDuration())
+matches y, a, b and
+c
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('hasThreadStorageDuration0')"><a name="hasThreadStorageDuration0Anchor">hasThreadStorageDuration</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="hasThreadStorageDuration0"><pre>Matches a variable declaration that has thread storage duration.
 
-Example matches z, but not x, z, or a.
-(matcher = varDecl(hasThreadStorageDuration())
+Given
 void f() {
   int x;
   static int y;
   thread_local int z;
 }
 int a;
+
+The matcher varDecl(hasThreadStorageDuration())
+matches z
+but does not match x, z or
+a
 </pre></td></tr>
 
 
@@ -5962,29 +8025,34 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isConstexpr0"><pre>Matches constexpr variable and function declarations,
        and if constexpr.
 
-Given:
+Given
   constexpr int foo = 42;
   constexpr int bar();
   void baz() { if constexpr(1 &gt; 0) {} }
-varDecl(isConstexpr())
-  matches the declaration of foo.
-functionDecl(isConstexpr())
-  matches the declaration of bar.
-ifStmt(isConstexpr())
-  matches the if statement in baz.
+
+The matcher varDecl(isConstexpr())
+matches foo.
+The matcher functionDecl(isConstexpr())
+matches bar.
+The matcher ifStmt(isConstexpr())
+matches if constexpr(1 &gt; 0) {}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('isConstinit0')"><a name="isConstinit0Anchor">isConstinit</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isConstinit0"><pre>Matches constinit variable declarations.
 
-Given:
+Given
   constinit int foo = 42;
   constinit const char* bar = "bar";
   int baz = 42;
   [[clang::require_constant_initialization]] int xyz = 42;
-varDecl(isConstinit())
-  matches the declaration of `foo` and `bar`, but not `baz` and `xyz`.
+
+The matcher varDecl(isConstinit())
+matches the declaration of foo
+and bar,
+but does not match baz or
+xyz.
 </pre></td></tr>
 
 
@@ -5998,6 +8066,14 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   extern int vb;  // Doesn't match, as it doesn't define the variable.
   void fa() {}
   void fb();  // Doesn't match, as it has no body.
+
+The matcher tagDecl(isDefinition())
+matches A
+The matcher varDecl(isDefinition())
+matches va
+The matcher functionDecl(isDefinition())
+matches fa
+
   @interface X
   - (void)ma; // Doesn't match, interface is declaration.
   @end
@@ -6005,6 +8081,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   - (void)ma {}
   @end
 
+The matcher objcMethodDecl(isDefinition())
+matches - (void)ma {}
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagDecl.html">TagDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMethodDecl.html">ObjCMethodDecl</a>&gt;
 </pre></td></tr>
@@ -6014,12 +8093,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isExceptionVariable0"><pre>Matches a variable declaration that is an exception variable from
 a C++ catch block, or an Objective-C statement.
 
-Example matches x (matcher = varDecl(isExceptionVariable())
+Given
 void f(int y) {
   try {
   } catch (int x) {
   }
 }
+
+The matcher varDecl(isExceptionVariable())
+matches x
 </pre></td></tr>
 
 
@@ -6030,8 +8112,9 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 Given
   template&lt;typename T&gt; void A(T t) { }
   template&lt;&gt; void A(int N) { }
-functionDecl(isExplicitTemplateSpecialization())
-  matches the specialization A&lt;int&gt;().
+
+The matcher functionDecl(isExplicitTemplateSpecialization())
+  matches the specialization template&lt;&gt; void A(int N) { }.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
 </pre></td></tr>
@@ -6040,17 +8123,21 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('isExternC1')"><a name="isExternC1Anchor">isExternC</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isExternC1"><pre>Matches extern "C" function or variable declarations.
 
-Given:
+Given
   extern "C" void f() {}
   extern "C" { void g() {} }
   void h() {}
   extern "C" int x = 1;
   extern "C" int y = 2;
   int z = 3;
-functionDecl(isExternC())
-  matches the declaration of f and g, but not the declaration of h.
-varDecl(isExternC())
-  matches the declaration of x and y, but not the declaration of z.
+
+The matcher functionDecl(isExternC())
+matches f
+and g.
+The matcher varDecl(isExternC())
+matches x
+and y,
+but does not match z.
 </pre></td></tr>
 
 
@@ -6058,8 +8145,11 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isInitCapture0"><pre>Matches a variable serving as the implicit variable for a lambda init-
 capture.
 
-Example matches x (matcher = varDecl(isInitCapture()))
-auto f = [x=3]() { return x; };
+Given
+auto f = [x = 3]() { return x; };
+
+The matcher varDecl(isInitCapture())
+matches x = 3.
 </pre></td></tr>
 
 
@@ -6074,21 +8164,24 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
   inline namespace m {}
   }
   inline int Foo = 5;
-functionDecl(isInline()) will match ::f().
-namespaceDecl(isInline()) will match n::m.
-varDecl(isInline()) will match Foo;
+
+The matcher functionDecl(isInline()) matches f.
+The matcher namespaceDecl(isInline()) matches m.
+The matcher varDecl(isInline()) matches Foo
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;</td><td class="name" onclick="toggle('isStaticLocal0')"><a name="isStaticLocal0Anchor">isStaticLocal</a></td><td></td></tr>
 <tr><td colspan="4" class="doc" id="isStaticLocal0"><pre>Matches a static variable with local scope.
 
-Example matches y (matcher = varDecl(isStaticLocal()))
+Given
 void f() {
   int x;
   static int y;
 }
 static int z;
+The matcher varDecl(isStaticLocal())
+matches y
 </pre></td></tr>
 
 
@@ -6096,15 +8189,15 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 <tr><td colspan="4" class="doc" id="isStaticStorageClass1"><pre>Matches variable/function declarations that have "static" storage
 class specifier ("static" keyword) written in the source.
 
-Given:
+Given
   static void f() {}
   static int i = 0;
   extern int j;
   int k;
-functionDecl(isStaticStorageClass())
-  matches the function declaration f.
-varDecl(isStaticStorageClass())
-  matches the variable declaration i.
+The matcher functionDecl(isStaticStorageClass())
+  matches f
+The matcher varDecl(isStaticStorageClass())
+  matches i
 </pre></td></tr>
 
 
@@ -6113,18 +8206,36 @@ <h2 id="narrowing-matchers">Narrowing Matchers</h2>
 member variable template instantiations.
 
 Given
-  template &lt;typename T&gt; class X {}; class A {}; X&lt;A&gt; x;
-or
-  template &lt;typename T&gt; class X {}; class A {}; template class X&lt;A&gt;;
-or
-  template &lt;typename T&gt; class X {}; class A {}; extern template class X&lt;A&gt;;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
-  matches the template instantiation of X&lt;A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches class X&lt;class A&gt;.
+  template &lt;typename T&gt; class X {};
+  class A {};
+  template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches template class X&lt;A&gt;
+  template &lt;typename T&gt; class X {};
+  class A {};
+  extern template class X&lt;A&gt;;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
+matches extern template class X&lt;A&gt;
 
 But given
-  template &lt;typename T&gt;  class X {}; class A {};
-  template &lt;&gt; class X&lt;A&gt; {}; X&lt;A&gt; x;
-cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
+  template &lt;typename T&gt;  class X {};
+  class A {};
+  template &lt;&gt; class X&lt;A&gt; {};
+  X&lt;A&gt; x;
+
+The matcher cxxRecordDecl(hasName("::X"),
+isTemplateInstantiation())
   does not match, as X&lt;A&gt; is an explicit template specialization.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;
@@ -6151,10 +8262,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;*&gt;</td><td class="name" onclick="toggle('binaryOperation0')"><a name="binaryOperation0Anchor">binaryOperation</a></td><td>Matcher&lt;*&gt;...Matcher&lt;*&gt;</td></tr>
 <tr><td colspan="4" class="doc" id="binaryOperation0"><pre>Matches nodes which can be used with binary operators.
 
-The code
-  var1 != var2;
-might be represented in the clang AST as a binaryOperator, a
-cxxOperatorCallExpr or a cxxRewrittenBinaryOperator, depending on
+A comparison of two expressions might be represented in the clang AST as a
+binaryOperator, a cxxOperatorCallExpr or a
+cxxRewrittenBinaryOperator, depending on
 
 * whether the types of var1 and var2 are fundamental (binaryOperator) or at
   least one is a class type (cxxOperatorCallExpr)
@@ -6168,12 +8278,6 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 compatible.
 
 Given
-  binaryOperation(
-    hasOperatorName("!="),
-    hasLHS(expr().bind("lhs")),
-    hasRHS(expr().bind("rhs"))
-  )
-matches each use of "!=" in:
   struct S{
       bool operator!=(const S&amp;) const;
   };
@@ -6187,25 +8291,28 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T&gt;
   void templ()
   {
-     1 != 2;
+     3 != 4;
      T() != S();
   }
   struct HasOpEq
   {
-      bool operator==(const HasOpEq &amp;) const;
+      friend bool
+      operator==(const HasOpEq &amp;, const HasOpEq&amp;) noexcept = default;
   };
 
   void inverse()
   {
-      HasOpEq s1;
-      HasOpEq s2;
-      if (s1 != s2)
+      HasOpEq e1;
+      HasOpEq e2;
+      if (e1 != e2)
           return;
   }
 
   struct HasSpaceship
   {
-      bool operator&lt;=&gt;(const HasOpEq &amp;) const;
+      friend bool
+      operator&lt;=&gt;(const HasSpaceship &amp;,
+                  const HasSpaceship&amp;) noexcept = default;
   };
 
   void use_spaceship()
@@ -6215,6 +8322,15 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
       if (s1 != s2)
           return;
   }
+
+
+The matcher binaryOperation(
+    hasOperatorName("!="),
+    hasLHS(expr().bind("lhs")),
+    hasRHS(expr().bind("rhs"))
+  )
+matches 1 != 2, S() != S(), 3 != 4,
+T() != S(), e1 != e2 and s1 != s2.
 </pre></td></tr>
 
 
@@ -6224,14 +8340,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Unlike anyOf, eachOf will generate a match result for each
 matching submatcher.
 
-For example, in:
-  class A { int a; int b; };
-The matcher:
-  cxxRecordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-                       has(fieldDecl(hasName("b")).bind("v"))))
-will generate two results binding "v", the first of which binds
-the field declaration of a, the second the field declaration of
-b.
+Given
+  void f(int a, int b);
+
+
+The matcher functionDecl(hasAnyParameter(
+eachOf(parmVarDecl(hasName("a")).bind("v"),
+       parmVarDecl(hasName("b")).bind("v"))))
+matches void f(int a, int b),
+with parmVarDecl(hasName("a")) matching a
+for one match,
+and with parmVarDecl(hasName("b")) matching
+b for the other match.
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6244,10 +8364,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 For example, in:
   class A { class B {}; class C {}; };
-The matcher:
-  cxxRecordDecl(hasName("::A"),
+
+The matcher
+cxxRecordDecl(hasName("::A"),
                 findAll(cxxRecordDecl(isDefinition()).bind("m")))
-will generate results for A, B and C.
+matches A three times,
+with cxxRecordDecl(isDefinition()).bind("m")
+matching A,
+B and C.
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6257,24 +8381,71 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="forEachDescendant0"><pre>Matches AST nodes that have descendant AST nodes that match the
 provided matcher.
 
-Example matches X, A, A::X, B, B::C, B::C::X
-  (matcher = cxxRecordDecl(forEachDescendant(cxxRecordDecl(hasName("X")))))
+Given
   class X {};
   class A { class X {}; };  // Matches A, because A::X is a class of name
                             // X inside A.
   class B { class C { class X {}; }; };
 
+The matcher
+cxxRecordDecl(forEachDescendant(cxxRecordDecl(hasName("X"))))
+matches X, A,
+B, class B::C
+and class B::C::X
+
 DescendantT must be an AST base type.
 
 As opposed to 'hasDescendant', 'forEachDescendant' will cause a match for
 each result that matches instead of only on the first one.
 
 Note: Recursively combined ForEachDescendant can cause many matches:
-  cxxRecordDecl(forEachDescendant(cxxRecordDecl(
-    forEachDescendant(cxxRecordDecl())
-  )))
-will match 10 times (plus injected class name matches) on:
-  class A { class B { class C { class D { class E {}; }; }; }; };
+  struct A {
+    struct B {
+      struct C {};
+      struct D {};
+    };
+  };
+
+
+The matcher cxxRecordDecl(forEachDescendant(cxxRecordDecl(
+    forEachDescendant(cxxRecordDecl().bind("inner"))
+  ).bind("middle")))
+will match 9 times:
+It matches the definition of A with the definition of
+B in the middle and the injected class name of
+B as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+C in the middle and the definition of
+B as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+C in the middle and the injected class name of
+B as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+B in the middle and the definition of
+D as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+B in the middle and the injected class name of
+D as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+C in the middle and the injected class name of
+C as the innermost cxxRecordDecl.
+
+It matches the definition of A with the definition of
+D in the middle and the injected class name of
+D as the innermost cxxRecordDecl.
+
+It matches the definition of B with the definition of
+C in the middle and the injected class name of
+C as the innermost cxxRecordDecl.
+
+It matches the definition of B with the definition of
+D in the middle and the injected class name of
+D as the innermost cxxRecordDecl.
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6284,17 +8455,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="forEach0"><pre>Matches AST nodes that have child AST nodes that match the
 provided matcher.
 
-Example matches X, Y, Y::X, Z::Y, Z::Y::X
-  (matcher = cxxRecordDecl(forEach(cxxRecordDecl(hasName("X")))
+Given
   class X {};
   class Y { class X {}; };  // Matches Y, because Y::X is a class of name X
                             // inside Y.
   class Z { class Y { class X {}; }; };  // Does not match Z.
 
+The matcher cxxRecordDecl(forEach(cxxRecordDecl(hasName("X"))))
+matches class X,
+class Y,
+class Y::X,
+class Z::Y::X and class Z::Y
+
 ChildT must be an AST base type.
 
 As opposed to 'has', 'forEach' will cause a match for each result that
-matches instead of only on the first one.
+  matches instead of only on the first one.
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6307,7 +8483,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
 void f() { if (true) { int x = 42; } }
 void g() { for (;;) { int x = 43; } }
-expr(integerLiteral(hasAncestor(ifStmt()))) matches 42, but not 43.
+
+The matcher expr(integerLiteral(hasAncestor(ifStmt())))
+matches 42
+but does not match 43
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6317,12 +8496,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasDescendant0"><pre>Matches AST nodes that have descendant AST nodes that match the
 provided matcher.
 
-Example matches X, Y, Z
-    (matcher = cxxRecordDecl(hasDescendant(cxxRecordDecl(hasName("X")))))
+Given
   class X {};  // Matches X, because X::X is a class of name X inside X.
   class Y { class X {}; };
   class Z { class Y { class X {}; }; };
 
+The matcher
+cxxRecordDecl(hasDescendant(cxxRecordDecl(hasName("X"))))
+matches class X {}, class Y { class X {}; }
+and class Z { class Y { class X {}; }; }.
+
 DescendantT must be an AST base type.
 
 Usable as: Any Matcher
@@ -6333,19 +8516,29 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="has0"><pre>Matches AST nodes that have child AST nodes that match the
 provided matcher.
 
-Example matches X, Y
-  (matcher = cxxRecordDecl(has(cxxRecordDecl(hasName("X")))
+Given
   class X {};  // Matches X, because X::X is a class of name X inside X.
   class Y { class X {}; };
   class Z { class Y { class X {}; }; };  // Does not match Z.
 
+The matcher cxxRecordDecl(has(cxxRecordDecl(hasName("X"))))
+matches class X {} three times,
+and class Y { class X {}; } two times.
+
 ChildT must be an AST base type.
 
 Usable as: Any Matcher
 Note that has is direct matcher, so it also matches things like implicit
 casts and paren casts. If you are matching with expr then you should
-probably consider using ignoringParenImpCasts like:
-has(ignoringParenImpCasts(expr())).
+probably consider using ignoringParenImpCasts:
+
+Given
+  int x =0;
+  double y = static_cast&lt;double&gt;(x);
+
+The matcher
+cxxStaticCastExpr(has(ignoringParenImpCasts(declRefExpr()))).
+matches static_cast&lt;double&gt;(x)
 </pre></td></tr>
 
 
@@ -6355,7 +8548,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
 void f() { for (;;) { int x = 42; if (true) { int x = 43; } } }
-compoundStmt(hasParent(ifStmt())) matches "{ int x = 43; }".
+
+The matcher compoundStmt(hasParent(ifStmt()))
+matches { int x = 43; }
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6369,7 +8564,7 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 which should match both are typically duplicated. This matcher
 removes the need for duplication.
 
-Given code
+Given
 struct ConstructorTakesInt
 {
   ConstructorTakesInt(int i) {}
@@ -6389,9 +8584,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   ConstructorTakesInt cti(42);
 }
 
+
 The matcher
-invocation(hasArgument(0, integerLiteral(equals(42))))
-matches the expression in both doCall and doConstruct
+expr(invocation(hasArgument(0, integerLiteral(equals(42)))))
+matches the expressions callTakesInt(42)
+and cti(42).
 </pre></td></tr>
 
 
@@ -6402,18 +8599,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Useful when additional information which may or may not present about a main
 matching node is desired.
 
-For example, in:
-  class Foo {
-    int bar;
-  }
-The matcher:
-  cxxRecordDecl(
-    optionally(has(
-      fieldDecl(hasName("bar")).bind("var")
-  ))).bind("record")
-will produce a result binding for both "record" and "var".
-The matcher will produce a "record" binding for even if there is no data
-member named "bar" in that class.
+Given
+  int a = 0;
+  int b;
+
+The matcher varDecl(optionally(hasInitializer(expr())))
+matches int a = 0 and int b.
 
 Usable as: Any Matcher
 </pre></td></tr>
@@ -6428,10 +8619,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
       int i = 3.0;
   }
 The matcher
-  traverse(TK_IgnoreUnlessSpelledInSource,
+traverse(TK_IgnoreUnlessSpelledInSource,
     varDecl(hasInitializer(floatLiteral().bind("init")))
   )
-matches the variable declaration with "init" bound to the "3.0".
+  matches int i = 3.0 with "init" bound to 3.0.
 </pre></td></tr>
 
 
@@ -6439,8 +8630,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition5"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 
@@ -6449,8 +8645,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 (binary or ternary).
 
 Example matches b
-  condition ? a : b
-  condition ?: b
+  void foo(bool condition, int a, int b) {
+    condition ? a : b;
+    condition ?: b;
+  }
+
+The matcher
+conditionalOperator(hasFalseExpression(expr().bind("false")))
+matches condition ? a : b,
+with expr() matching b.
+The matcher
+binaryConditionalOperator(hasFalseExpression(expr().bind("false")))
+matches condition ?: b,
+with expr() matching b.
 </pre></td></tr>
 
 
@@ -6458,16 +8665,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasTrueExpression0"><pre>Matches the true branch expression of a conditional operator.
 
 Example 1 (conditional ternary operator): matches a
-  condition ? a : b
+Given
+  void foo(bool condition, int a, int b) {
+    condition ? a : b;
+  }
+
+The matcher
+conditionalOperator(hasTrueExpression(expr().bind("true")))
+matches condition ? a : b,
+with expr() matching a.
 
 Example 2 (conditional binary operator): matches opaqueValueExpr(condition)
-  condition ?: b
+Given
+  void foo(bool condition, int a, int b) {
+    condition ?: b;
+  }
+
+The matcher binaryConditionalOperator(hasTrueExpression(expr()))
+matches condition ?: b,
+with expr() matching conditoin.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration15')"><a name="hasDeclaration15Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration15"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -6477,17 +8699,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -6505,7 +8735,7 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   int i[5];
   void f() { i[1] = 42; }
-arraySubscriptExpression(hasBase(implicitCastExpr(
+The matcher arraySubscriptExpr(hasBase(implicitCastExpr(
     hasSourceExpression(declRefExpr()))))
   matches i[1] with the declRefExpr() matching i
 </pre></td></tr>
@@ -6517,7 +8747,7 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   int i[5];
   void f() { i[1] = 42; }
-arraySubscriptExpression(hasIndex(integerLiteral()))
+The matcher arraySubscriptExpr(hasIndex(integerLiteral()))
   matches i[1] with the integerLiteral() matching 1
 </pre></td></tr>
 
@@ -6525,16 +8755,30 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArraySubscriptExpr.html">ArraySubscriptExpr</a>&gt;</td><td class="name" onclick="toggle('hasLHS3')"><a name="hasLHS3Anchor">hasLHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLHS3"><pre>Matches the left hand side of binary operator expressions.
 
-Example matches a (matcher = binaryOperator(hasLHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasLHS(expr().bind("lhs")))
+matches a || b,
+with expr()
+matching a.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArraySubscriptExpr.html">ArraySubscriptExpr</a>&gt;</td><td class="name" onclick="toggle('hasRHS3')"><a name="hasRHS3Anchor">hasRHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRHS3"><pre>Matches the right hand side of binary operator expressions.
 
-Example matches b (matcher = binaryOperator(hasRHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasRHS(expr().bind("rhs")))
+matches a || b,
+with expr()
+matching b.
 </pre></td></tr>
 
 
@@ -6546,8 +8790,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   struct A {};
   A a[7];
   int b[7];
-arrayType(hasElementType(builtinType()))
-  matches "int b[7]"
+
+
+The matcher arrayType(hasElementType(builtinType()))
+int[7]
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArrayType.html">ArrayType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ComplexType.html">ComplexType</a>&gt;
 </pre></td></tr>
@@ -6559,8 +8805,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   _Atomic(int) i;
   _Atomic(float) f;
-atomicType(hasValueType(isInteger()))
- matches "_Atomic(int) i"
+The matcher atomicType(hasValueType(isInteger()))
+_Atomic(int).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AtomicType.html">AtomicType</a>&gt;
 </pre></td></tr>
@@ -6575,8 +8821,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   auto a = 1;
   auto b = 2.0;
-autoType(hasDeducedType(isInteger()))
-  matches "auto a"
+
+The matcher
+varDecl(hasType(autoType(hasDeducedType(isInteger()))))
+matches auto a = 1, but does not match auto b = 2.0.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AutoType.html">AutoType</a>&gt;
 </pre></td></tr>
@@ -6588,21 +8836,54 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   namespace X { void b(); }
   using X::b;
-usingDecl(hasAnyUsingShadowDecl(hasName("b"))))
-  matches using X::b </pre></td></tr>
+
+The matcher usingDecl(hasAnyUsingShadowDecl(hasName("b")))
+  matches using X::b
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasEitherOperand0')"><a name="hasEitherOperand0Anchor">hasEitherOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasEitherOperand0"><pre>Matches if either the left hand side or the right hand side of a
 binary operator or fold expression matches.
+
+Given
+  struct S {};
+  bool operator ==(const S&amp;, const S&amp;);
+
+  void f(int a, const S&amp;lhs, const S&amp;rhs) {
+      a + 0;
+      lhs == rhs;
+      lhs != rhs;
+  }
+
+  template &lt;typename ...Ts&gt;
+  auto sum(Ts... args) {
+    return (0 + ... + args);
+  }
+
+
+The matcher binaryOperator(hasEitherOperand(integerLiteral()))
+matches a + 0.
+The matcher cxxOperatorCallExpr(hasEitherOperand(declRefExpr(to(
+parmVarDecl(hasName("lhs")))))) matches lhs == rhs and
+lhs != rhs.
+The matcher cxxFoldExpr(hasEitherOperand(integerLiteral()))
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasLHS0')"><a name="hasLHS0Anchor">hasLHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLHS0"><pre>Matches the left hand side of binary operator expressions.
 
-Example matches a (matcher = binaryOperator(hasLHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasLHS(expr().bind("lhs")))
+matches a || b,
+with expr()
+matching a.
 </pre></td></tr>
 
 
@@ -6610,27 +8891,40 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperands0"><pre>Matches if both matchers match with opposite sides of the binary operator
 or fold expression.
 
-Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
-                                             integerLiteral(equals(2)))
-  1 + 2 // Match
-  2 + 1 // Match
-  1 + 1 // No match
-  2 + 2 // No match
+Given
+void foo() {
+  1 + 2; // Match
+  2 + 1; // Match
+  1 + 1; // No match
+  2 + 2; // No match
+}
+The matcher binaryOperator(hasOperands(integerLiteral(equals(1)),
+                                            integerLiteral(equals(2))))
+matches 1 + 2 and 2 + 1,
+but does not match 1 + 1
+or 2 + 2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BinaryOperator.html">BinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasRHS0')"><a name="hasRHS0Anchor">hasRHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRHS0"><pre>Matches the right hand side of binary operator expressions.
 
-Example matches b (matcher = binaryOperator(hasRHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasRHS(expr().bind("rhs")))
+matches a || b,
+with expr()
+matching b.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BindingDecl.html">BindingDecl</a>&gt;</td><td class="name" onclick="toggle('forDecomposition0')"><a name="forDecomposition0Anchor">forDecomposition</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="forDecomposition0"><pre>Matches the DecompositionDecl the binding belongs to.
 
-For example, in:
+Given
 void foo()
 {
     int arr[3];
@@ -6638,10 +8932,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
     f = 42;
 }
-The matcher:
-  bindingDecl(hasName("f"),
-                forDecomposition(decompositionDecl())
-matches 'f' in 'auto &amp;[f, s, t]'.
+
+The matcher bindingDecl(hasName("f"),
+                forDecomposition(decompositionDecl()))
+matches f in 'auto &amp;[f, s, t]'.
 </pre></td></tr>
 
 
@@ -6653,23 +8947,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x, int y, int z) {} };
-cxxMethodDecl(hasAnyParameter(hasName("y")))
-  matches f(int x, int y, int z) {}
+
+The matcher cxxMethodDecl(hasAnyParameter(hasName("y")))
+  matches f
 with hasAnyParameter(...)
   matching int y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
+
 the matcher objcMethodDecl(hasAnyParameter(hasName("y")))
-matches the declaration of method f with hasParameter
+  matches the declaration of method f with hasParameter
 matching y.
 
 For blocks, given
   b = ^(int y) { printf("%d", y) };
 
+
 the matcher blockDecl(hasAnyParameter(hasName("y")))
-matches the declaration of the block b with hasParameter
+  matches the declaration of the block b with hasParameter
 matching y.
 </pre></td></tr>
 
@@ -6680,15 +8977,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x) {} };
-cxxMethodDecl(hasParameter(0, hasType(varDecl())))
-  matches f(int x) {}
+
+The matcher
+cxxMethodDecl(hasParameter(0, hasType(asString("int"))))
+matches f
 with hasParameter(...)
-  matching int x
+matching int x.
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
-the matcher objcMethodDecl(hasParameter(0, hasName("y")))
+
+The matcher objcMethodDecl(hasParameter(0, hasName("y")))
 matches the declaration of method f with hasParameter
 matching y.
 </pre></td></tr>
@@ -6697,19 +8997,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc0')"><a name="hasTypeLoc0Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc0"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -6728,10 +9035,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int *a;
-  int const *b;
-  float const *f;
-pointerType(pointee(isConstQualified(), isInteger()))
-  matches "int const *b"
+  const int *b;
+  int * const c = nullptr;
+  const float *f;
+
+The matcher pointerType(pointee(isConstQualified(), isInteger()))
+matches const int *,
+but does not match int * const
+or const float *.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
@@ -6741,19 +9052,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc1')"><a name="hasTypeLoc1Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc1"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -6776,21 +9094,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
 declaration of x.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              cxxRecordDecl(hasName("X"))))
  class X {};
  void y(X &amp;x) { x; X z; }
  class Y { friend class X; };
  class Z : public virtual X {};
 
-Example matches class Derived
-(matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z.
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X.
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches
+class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
+
+Given
 class Base {};
 class Derived : Base {};
 
+The matcher
+cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base")))))
+matches class Derived : Base {}.
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FriendDecl.html">FriendDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;,
 Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -6800,17 +9128,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasType4"><pre>Matches if the expression's or declaration's type matches a type
 matcher.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and U (matcher = typedefDecl(hasType(asString("int")))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              asString("class X")))
+Exmaple
  class X {};
  void y(X &amp;x) { x; X z; }
  typedef int U;
  class Y { friend class X; };
  class Z : public virtual X {};
+
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z
+The matcher typedefDecl(hasType(asString("int")))
+matches typedef int U
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
 </pre></td></tr>
 
 
@@ -6820,8 +9156,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   void f(int i);
   int y;
-  f(y);
-callExpr(
+  void foo() {
+    f(y);
+  }
+The matcher callExpr(
   forEachArgumentWithParam(
     declRefExpr(to(varDecl(hasName("y")))),
     parmVarDecl(hasType(isInteger()))
@@ -6844,14 +9182,15 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   void f(int i);
-  int y;
-  f(y);
-  void (*f_ptr)(int) = f;
-  f_ptr(y);
-callExpr(
+  void foo(int y) {
+    f(y);
+    void (*f_ptr)(int) = f;
+    f_ptr(y);
+  }
+The matcher callExpr(
   forEachArgumentWithParamType(
     declRefExpr(to(varDecl(hasName("y")))),
-    qualType(isInteger()).bind("type)
+    qualType(isInteger()).bind("type")
 ))
   matches f(y) and f_ptr(y)
 with declRefExpr(...)
@@ -6866,17 +9205,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 expression, or an ObjC-message-send expression.
 
 Given
-  void x(int, int, int) { int y; x(1, y, 42); }
-callExpr(hasAnyArgument(declRefExpr()))
-  matches x(1, y, 42)
-with hasAnyArgument(...)
+  void x(int, int, int) { int y = 42; x(1, y, 42); }
+The matcher
+callExpr(hasAnyArgument(ignoringImplicit(declRefExpr()))) matches
+x(1, y, 42) with hasAnyArgument(...)
   matching y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
   void foo(I *i) { [i f:12]; }
+
+The matcher
 objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))
-  matches [i f:12]
+matches [i f:12]
 </pre></td></tr>
 
 
@@ -6884,15 +9225,17 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasArgument1"><pre>Matches the n'th argument of a call expression or a constructor
 call expression.
 
-Example matches y in x(y)
-    (matcher = callExpr(hasArgument(0, declRefExpr())))
+Given
   void x(int) { int y; x(y); }
+The matcher callExpr(hasArgument(0, declRefExpr().bind("arg")))
+matches x(y),
+with declRefExpr() matching y.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration13')"><a name="hasDeclaration13Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration13"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -6902,17 +9245,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -6929,10 +9280,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class A { A() : i(42), j(42) {} int i; int j; };
-cxxConstructorDecl(forEachConstructorInitializer(
-  forField(decl().bind("x"))
-))
-  will trigger two matches, binding for 'i' and 'j' respectively.
+
+The matcher cxxConstructorDecl(forEachConstructorInitializer(
+  forField(fieldDecl().bind("x"))))
+matches the constructor of A twice, with
+fieldDecl() matching i and
+j respectively.
 </pre></td></tr>
 
 
@@ -6944,10 +9297,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     Foo() : foo_(1) { }
     int foo_;
   };
-cxxRecordDecl(has(cxxConstructorDecl(
+
+The matcher cxxRecordDecl(has(cxxConstructorDecl(
   hasAnyConstructorInitializer(anything())
 )))
-  record matches Foo, hasAnyConstructorInitializer matches foo_(1)
+matches Foo, hasAnyConstructorInitializer matches foo_(1)
 </pre></td></tr>
 
 
@@ -6959,9 +9313,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     Foo() : foo_(1) { }
     int foo_;
   };
+
+The matcher
 cxxRecordDecl(has(cxxConstructorDecl(hasAnyConstructorInitializer(
     forField(hasName("foo_"))))))
-  matches Foo
+matches Foo
 with forField matching foo_
 </pre></td></tr>
 
@@ -6969,19 +9325,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc2')"><a name="hasTypeLoc2Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc2"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -7002,9 +9365,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     Foo() : foo_(1) { }
     int foo_;
   };
+
+The matcher
 cxxRecordDecl(has(cxxConstructorDecl(hasAnyConstructorInitializer(
     withInitializer(integerLiteral(equals(1)))))))
-  matches Foo
+matches Foo
 with withInitializer matching (1)
 </pre></td></tr>
 
@@ -7019,11 +9384,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     int m;
     int f(X x) { x.m; return m; }
   };
+
+
+The matcher
 memberExpr(hasObjectExpression(hasType(cxxRecordDecl(hasName("X")))))
-  matches `x.m`, but not `m`; however,
-memberExpr(hasObjectExpression(hasType(pointsTo(
-     cxxRecordDecl(hasName("X"))))))
-  matches `m` (aka. `this-&gt;m`), but not `x.m`.
+matches x.m, but not m; however,
+The matcher memberExpr(hasObjectExpression(hasType(pointsTo(
+cxxRecordDecl(hasName("X"))))))
+matches m (aka. this-&gt;m), but not x.m.
 </pre></td></tr>
 
 
@@ -7033,12 +9401,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   class Y { void x() { this-&gt;x(); x(); Y y; y.x(); } };
   void f() { f(); }
-callExpr(callee(expr()))
-  matches this-&gt;x(), x(), y.x(), f()
-with callee(...)
-  matching this-&gt;x, x, y.x, f respectively
+
+The matcher callExpr(callee(expr().bind("callee")))
+matches this-&gt;x(), x(), y.x(), f()
+with expr() inside of callee
+matching this-&gt;x, x,
+y.x, f respectively
 
 Given
+  struct Dummy {};
+  // makes sure there is a callee, otherwise there would be no callee,
+  // just a builtin operator
+  Dummy operator+(Dummy, Dummy);
+  // not defining a '*' operator
+
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -7048,10 +9424,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
-cxxFoldExpr(callee(expr()))
-  matches (args * ... * 1)
-with callee(...)
-  matching *
+
+The matcher cxxFoldExpr(callee(expr().bind("op")))
+matches (0 + ... + args)
+with callee(...) matching *,
+but does not match (args * ... * 1).
+A CXXFoldExpr only has an UnresolvedLookupExpr as a callee.
+When there are no define operators that could be used instead of builtin
+ones, then there will be no callee .
 
 Note: Callee cannot take the more general internal::Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;
 because this introduces ambiguous overloads with calls to Callee taking a
@@ -7063,16 +9443,37 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('hasEitherOperand2')"><a name="hasEitherOperand2Anchor">hasEitherOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasEitherOperand2"><pre>Matches if either the left hand side or the right hand side of a
 binary operator or fold expression matches.
+
+Given
+  struct S {};
+  bool operator ==(const S&amp;, const S&amp;);
+
+  void f(int a, const S&amp;lhs, const S&amp;rhs) {
+      a + 0;
+      lhs == rhs;
+      lhs != rhs;
+  }
+
+  template &lt;typename ...Ts&gt;
+  auto sum(Ts... args) {
+    return (0 + ... + args);
+  }
+
+
+The matcher binaryOperator(hasEitherOperand(integerLiteral()))
+matches a + 0.
+The matcher cxxOperatorCallExpr(hasEitherOperand(declRefExpr(to(
+parmVarDecl(hasName("lhs")))))) matches lhs == rhs and
+lhs != rhs.
+The matcher cxxFoldExpr(hasEitherOperand(integerLiteral()))
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('hasFoldInit0')"><a name="hasFoldInit0Anchor">hasFoldInit</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMacher</td></tr>
 <tr><td colspan="4" class="doc" id="hasFoldInit0"><pre>Matches the operand that does not contain the parameter pack.
 
-Example matches `(0 + ... + args)` and `(args * ... * 1)`
-    (matcher = cxxFoldExpr(hasFoldInit(expr())))
-  with hasFoldInit(...)
-    matching `0` and `1` respectively
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -7082,14 +9483,27 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
+
+
+The matcher cxxFoldExpr(hasFoldInit(expr().bind("init")))
+matches (0 + ... + args) and (args * ... * 1)
+with hasFoldInit(expr().bind("init")) matching
+0 and 1.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('hasLHS4')"><a name="hasLHS4Anchor">hasLHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLHS4"><pre>Matches the left hand side of binary operator expressions.
 
-Example matches a (matcher = binaryOperator(hasLHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasLHS(expr().bind("lhs")))
+matches a || b,
+with expr()
+matching a.
 </pre></td></tr>
 
 
@@ -7097,22 +9511,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperands2"><pre>Matches if both matchers match with opposite sides of the binary operator
 or fold expression.
 
-Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
-                                             integerLiteral(equals(2)))
-  1 + 2 // Match
-  2 + 1 // Match
-  1 + 1 // No match
-  2 + 2 // No match
+Given
+void foo() {
+  1 + 2; // Match
+  2 + 1; // Match
+  1 + 1; // No match
+  2 + 2; // No match
+}
+The matcher binaryOperator(hasOperands(integerLiteral(equals(1)),
+                                            integerLiteral(equals(2))))
+matches 1 + 2 and 2 + 1,
+but does not match 1 + 1
+or 2 + 2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('hasPattern0')"><a name="hasPattern0Anchor">hasPattern</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMacher</td></tr>
 <tr><td colspan="4" class="doc" id="hasPattern0"><pre>Matches the operand that contains the parameter pack.
 
-Example matches `(0 + ... + args)`
-    (matcher = cxxFoldExpr(hasPattern(expr())))
-  with hasPattern(...)
-    matching `args`
+Given
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -7122,14 +9539,27 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
+
+
+The matcher cxxFoldExpr(hasPattern(expr().bind("pattern")))
+matches (0 + ... + args) and (args * ... * 1),
+with hasPattern(expr().bind("pattern")) matching
+args two times.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFoldExpr.html">CXXFoldExpr</a>&gt;</td><td class="name" onclick="toggle('hasRHS4')"><a name="hasRHS4Anchor">hasRHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRHS4"><pre>Matches the right hand side of binary operator expressions.
 
-Example matches b (matcher = binaryOperator(hasRHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasRHS(expr().bind("rhs")))
+matches a || b,
+with expr()
+matching b.
 </pre></td></tr>
 
 
@@ -7140,27 +9570,32 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXForRangeStmt.html">CXXForRangeStmt</a>&gt;</td><td class="name" onclick="toggle('hasInitStatement2')"><a name="hasInitStatement2Anchor">hasInitStatement</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasInitStatement2"><pre>Matches selection statements with initializer.
 
-Given:
+Given
+ struct vec { int* begin(); int* end(); };
+ int foobar();
+ vec&amp; get_range();
  void foo() {
    if (int i = foobar(); i &gt; 0) {}
    switch (int i = foobar(); i) {}
@@ -7171,51 +9606,71 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
    switch (foobar()) {}
    for (auto&amp; x : get_range()) {}
  }
-ifStmt(hasInitStatement(anything()))
-  matches the if statement in foo but not in bar.
-switchStmt(hasInitStatement(anything()))
-  matches the switch statement in foo but not in bar.
-cxxForRangeStmt(hasInitStatement(anything()))
-  matches the range for statement in foo but not in bar.
+
+The matcher ifStmt(hasInitStatement(anything()))
+  matches the if statement if (int i = foobar(); i &gt; 0) {}
+  in foo but not if (foobar() &gt; 0) {} in bar.
+The matcher switchStmt(hasInitStatement(anything()))
+  matches the switch statement switch (int i = foobar(); i) {}
+  in foo but not switch (foobar()) {} in bar.
+The matcher cxxForRangeStmt(hasInitStatement(anything()))
+  matches the range for statement
+  for (auto&amp; a = get_range(); auto&amp; x : a) {} in foo
+  but not for (auto&amp; x : get_range()) {} in bar.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXForRangeStmt.html">CXXForRangeStmt</a>&gt;</td><td class="name" onclick="toggle('hasLoopVariable0')"><a name="hasLoopVariable0Anchor">hasLoopVariable</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1VarDecl.html">VarDecl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLoopVariable0"><pre>Matches the initialization statement of a for loop.
 
-Example:
-    forStmt(hasLoopVariable(anything()))
-matches 'int x' in
+Given
+  void foo() {
+    int a[42] = {};
     for (int x : a) { }
+  }
+
+The matcher cxxForRangeStmt(hasLoopVariable(anything()))
+matches for (int x : a) { }
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXForRangeStmt.html">CXXForRangeStmt</a>&gt;</td><td class="name" onclick="toggle('hasRangeInit0')"><a name="hasRangeInit0Anchor">hasRangeInit</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRangeInit0"><pre>Matches the range initialization statement of a for loop.
 
-Example:
-    forStmt(hasRangeInit(anything()))
-matches 'a' in
+Given
+  void foo() {
+    int a[42] = {};
     for (int x : a) { }
+  }
+
+The matcher cxxForRangeStmt(hasRangeInit(anything()))
+matches for (int x : a) { }
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc3')"><a name="hasTypeLoc3Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc3"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -7235,13 +9690,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   class Y { public: void m(); };
   Y g();
-  class X : public Y { void g(); };
+  class X : public Y { public: void g(); };
   void z(Y y, X x) { y.m(); x.m(); x.g(); (g()).m(); }
-cxxMemberCallExpr(onImplicitObjectArgument(hasType(
+
+The matcher cxxMemberCallExpr(onImplicitObjectArgument(hasType(
     cxxRecordDecl(hasName("Y")))))
-  matches `y.m()`, `x.m()` and (g()).m(), but not `x.g()`.
-cxxMemberCallExpr(on(callExpr()))
-  does not match `(g()).m()`, because the parens are not ignored.
+matches y.m(), x.m() and (g()).m()
+but does not match x.g().
+The matcher cxxMemberCallExpr(on(callExpr()))
+matches (g()).m(), because the parens are ignored.
+FIXME: should they be ignored? (ignored bc of `on`)
 
 FIXME: Overload to allow directly matching types?
 </pre></td></tr>
@@ -7256,12 +9714,15 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   Y g();
   class X : public Y {};
   void z(Y y, X x) { y.m(); (g()).m(); x.m(); }
+
+The matcher
 cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("Y")))))
-  matches `y.m()` and `(g()).m()`.
+  matches y.m() and (g()).m().
+The matcher
 cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("X")))))
-  matches `x.m()`.
-cxxMemberCallExpr(on(callExpr()))
-  matches `(g()).m()`.
+  matches x.m().
+The matcher cxxMemberCallExpr(on(callExpr()))
+  matches (g()).m().
 
 FIXME: Overload to allow directly matching types?
 </pre></td></tr>
@@ -7269,24 +9730,35 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMemberCallExpr.html">CXXMemberCallExpr</a>&gt;</td><td class="name" onclick="toggle('thisPointerType1')"><a name="thisPointerType1Anchor">thisPointerType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="thisPointerType1"><pre>Overloaded to match the type's declaration.
+
+Given
+  class Y { public: void m(); };
+  class X : public Y { public: void g(); };
+  void z() { Y y; y.m(); Y *p; p-&gt;m(); X x; x.m(); x.g(); }
+
+The matcher cxxMemberCallExpr(thisPointerType(
+    cxxRecordDecl(hasName("Y"))))
+  matches y.m(), p-&gt;m() and x.m().
+The matcher cxxMemberCallExpr(thisPointerType(
+    cxxRecordDecl(hasName("X"))))
+  matches x.g().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMemberCallExpr.html">CXXMemberCallExpr</a>&gt;</td><td class="name" onclick="toggle('thisPointerType0')"><a name="thisPointerType0Anchor">thisPointerType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="thisPointerType0"><pre>Matches if the type of the expression's implicit object argument either
-matches the InnerMatcher, or is a pointer to a type that matches the
+  matches the InnerMatcher, or is a pointer to a type that matches the
 InnerMatcher.
 
 Given
-  class Y { public: void m(); };
-  class X : public Y { void g(); };
-  void z() { Y y; y.m(); Y *p; p-&gt;m(); X x; x.m(); x.g(); }
-cxxMemberCallExpr(thisPointerType(hasDeclaration(
-    cxxRecordDecl(hasName("Y")))))
-  matches `y.m()`, `p-&gt;m()` and `x.m()`.
-cxxMemberCallExpr(thisPointerType(hasDeclaration(
-    cxxRecordDecl(hasName("X")))))
-  matches `x.g()`.
+  class Y { public: void m() const; };
+  class X : public Y { public: void g(); };
+  void z() { const Y y; y.m(); const Y *p; p-&gt;m(); X x; x.m(); x.g(); }
+
+The matcher
+cxxMemberCallExpr(thisPointerType(isConstQualified()))
+matches y.m(), x.m() and p-&gt;m(),
+but not x.g().
 </pre></td></tr>
 
 
@@ -7298,19 +9770,27 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   class A { virtual void f(); };
   class B : public A { void f(); };
   class C : public B { void f(); };
-cxxMethodDecl(ofClass(hasName("C")),
-              forEachOverridden(cxxMethodDecl().bind("b"))).bind("d")
-  matches once, with "b" binding "A::f" and "d" binding "C::f" (Note
-  that B::f is not overridden by C::f).
+
+The matcher cxxMethodDecl(ofClass(hasName("C")),
+              forEachOverridden(cxxMethodDecl().bind("b")))
+matches void f() of C ,
+with cxxMethodDecl() matching
+virtual void f() of A ,
+but the matcher does not match void f() of B because
+it is not overridden by C::f.
 
 The check can produce multiple matches in case of multiple inheritance, e.g.
   class A1 { virtual void f(); };
   class A2 { virtual void f(); };
   class C : public A1, public A2 { void f(); };
-cxxMethodDecl(ofClass(hasName("C")),
-              forEachOverridden(cxxMethodDecl().bind("b"))).bind("d")
-  matches twice, once with "b" binding "A1::f" and "d" binding "C::f", and
-  once with "b" binding "A2::f" and "d" binding "C::f".
+
+The matcher cxxMethodDecl(ofClass(hasName("C")),
+              forEachOverridden(cxxMethodDecl().bind("b")))
+matches void f() of C with the inner
+cxxMethodDecl() matching virtual void f()
+inside of A1 , and void f() of C with the inner
+cxxMethodDecl() matching virtual void f()
+inside of A2.
 </pre></td></tr>
 
 
@@ -7322,40 +9802,52 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 FIXME: What other kind of declarations would we need to generalize
 this to?
 
-Example matches A() in the last line
-    (matcher = cxxConstructExpr(hasDeclaration(cxxMethodDecl(
-        ofClass(hasName("A"))))))
+Given
   class A {
    public:
     A();
+    void foo();
   };
-  A a = A();
+
+The matcher cxxMethodDecl(ofClass(hasName("A")))
+matches A() and void foo().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('hasAnyPlacementArg0')"><a name="hasAnyPlacementArg0Anchor">hasAnyPlacementArg</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasAnyPlacementArg0"><pre>Matches any placement new expression arguments.
 
-Given:
+Given
+  void* operator new(decltype(sizeof(void*)), void*);
+  struct MyClass { int x; };
+  unsigned char Storage[sizeof(MyClass) * 10];
   MyClass *p1 = new (Storage) MyClass();
-cxxNewExpr(hasAnyPlacementArg(anything()))
-  matches the expression 'new (Storage, 16) MyClass()'.
+
+
+The matcher cxxNewExpr(hasAnyPlacementArg(anything()))
+matches new (Storage) MyClass().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('hasArraySize0')"><a name="hasArraySize0Anchor">hasArraySize</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasArraySize0"><pre>Matches array new expressions with a given array size.
 
-Given:
+Given
+  void* operator new(decltype(sizeof(void*)));
+  struct MyClass { int x; };
   MyClass *p1 = new MyClass[10];
-cxxNewExpr(hasArraySize(integerLiteral(equals(10))))
-  matches the expression 'new MyClass[10]'.
+
+
+The matcher
+cxxNewExpr(hasArraySize(
+            ignoringImplicit(integerLiteral(equals(10)))))
+matches new MyClass[10].
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration12')"><a name="hasDeclaration12Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration12"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -7365,17 +9857,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -7390,29 +9890,42 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('hasPlacementArg0')"><a name="hasPlacementArg0Anchor">hasPlacementArg</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasPlacementArg0"><pre>Matches placement new expression arguments.
 
-Given:
-  MyClass *p1 = new (Storage, 16) MyClass();
-cxxNewExpr(hasPlacementArg(1, integerLiteral(equals(16))))
-  matches the expression 'new (Storage, 16) MyClass()'.
+Given
+  void *operator new(decltype(sizeof(void*)), int, void*);
+  struct MyClass { int x; };
+  unsigned char Storage[sizeof(MyClass) * 10];
+  MyClass *p1 = new (16, Storage) MyClass();
+
+
+The matcher cxxNewExpr(hasPlacementArg(0,
+                      integerLiteral(equals(16))))
+matches new (16, Storage) MyClass().
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc4')"><a name="hasTypeLoc4Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc4"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -7428,14 +9941,45 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('hasEitherOperand1')"><a name="hasEitherOperand1Anchor">hasEitherOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasEitherOperand1"><pre>Matches if either the left hand side or the right hand side of a
 binary operator or fold expression matches.
+
+Given
+  struct S {};
+  bool operator ==(const S&amp;, const S&amp;);
+
+  void f(int a, const S&amp;lhs, const S&amp;rhs) {
+      a + 0;
+      lhs == rhs;
+      lhs != rhs;
+  }
+
+  template &lt;typename ...Ts&gt;
+  auto sum(Ts... args) {
+    return (0 + ... + args);
+  }
+
+
+The matcher binaryOperator(hasEitherOperand(integerLiteral()))
+matches a + 0.
+The matcher cxxOperatorCallExpr(hasEitherOperand(declRefExpr(to(
+parmVarDecl(hasName("lhs")))))) matches lhs == rhs and
+lhs != rhs.
+The matcher cxxFoldExpr(hasEitherOperand(integerLiteral()))
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('hasLHS1')"><a name="hasLHS1Anchor">hasLHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLHS1"><pre>Matches the left hand side of binary operator expressions.
 
-Example matches a (matcher = binaryOperator(hasLHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasLHS(expr().bind("lhs")))
+matches a || b,
+with expr()
+matching a.
 </pre></td></tr>
 
 
@@ -7443,44 +9987,64 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperands1"><pre>Matches if both matchers match with opposite sides of the binary operator
 or fold expression.
 
-Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
-                                             integerLiteral(equals(2)))
-  1 + 2 // Match
-  2 + 1 // Match
-  1 + 1 // No match
-  2 + 2 // No match
+Given
+void foo() {
+  1 + 2; // Match
+  2 + 1; // Match
+  1 + 1; // No match
+  2 + 2; // No match
+}
+The matcher binaryOperator(hasOperands(integerLiteral(equals(1)),
+                                            integerLiteral(equals(2))))
+matches 1 + 2 and 2 + 1,
+but does not match 1 + 1
+or 2 + 2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('hasRHS1')"><a name="hasRHS1Anchor">hasRHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRHS1"><pre>Matches the right hand side of binary operator expressions.
 
-Example matches b (matcher = binaryOperator(hasRHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasRHS(expr().bind("rhs")))
+matches a || b,
+with expr()
+matching b.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXOperatorCallExpr.html">CXXOperatorCallExpr</a>&gt;</td><td class="name" onclick="toggle('hasUnaryOperand1')"><a name="hasUnaryOperand1Anchor">hasUnaryOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasUnaryOperand1"><pre>Matches if the operand of a unary operator matches.
 
-Example matches true (matcher = hasUnaryOperand(
-                                  cxxBoolLiteral(equals(true))))
-  !true
+void foo() {
+  !true;
+}
+
+The matcher
+unaryOperator(hasUnaryOperand(cxxBoolLiteral(equals(true))))
+matches !true.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasAnyBase0')"><a name="hasAnyBase0Anchor">hasAnyBase</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt; BaseSpecMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasAnyBase0"><pre>Matches C++ classes that have a direct or indirect base matching BaseSpecMatcher.
 
-Example:
-matcher hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
-  class Foo;
+Given
+  class Foo {};
   class Bar : Foo {};
   class Baz : Bar {};
-  class SpecialBase;
+  class SpecialBase {};
   class Proxy : SpecialBase {};  // matches Proxy
   class IndirectlyDerived : Proxy {};  //matches IndirectlyDerived
 
+
+The matcher
+cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase")))))
+matches Proxy and IndirectlyDerived
 FIXME: Refactor this and isDerivedFrom to reuse implementation.
 </pre></td></tr>
 
@@ -7488,26 +10052,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasDirectBase0')"><a name="hasDirectBase0Anchor">hasDirectBase</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt; BaseSpecMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDirectBase0"><pre>Matches C++ classes that have a direct base matching BaseSpecMatcher.
 
-Example:
-matcher hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
-  class Foo;
+Given
+  class Foo {};
   class Bar : Foo {};
   class Baz : Bar {};
-  class SpecialBase;
+  class SpecialBase {};
   class Proxy : SpecialBase {};  // matches Proxy
   class IndirectlyDerived : Proxy {};  // doesn't match
+
+The matcher
+cxxRecordDecl(hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase")))))
+matches Proxy
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('hasMethod0')"><a name="hasMethod0Anchor">hasMethod</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXMethodDecl.html">CXXMethodDecl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasMethod0"><pre>Matches the first method of a class or struct that satisfies InnerMatcher.
 
-Given:
+Given
   class A { void func(); };
   class B { void member(); };
 
-cxxRecordDecl(hasMethod(hasName("func"))) matches the declaration of
-A but not B.
+
+The matcher cxxRecordDecl(hasMethod(hasName("func")))
+matches the declaration of class A { void func(); }
+but does not match class B { void member(); }
 </pre></td></tr>
 
 
@@ -7519,22 +10088,29 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Note that a class is not considered to be derived from itself.
 
 Example matches Y, Z, C (Base == hasName("X"))
-  class X;
+  class X {};
   class Y : public X {};  // directly derived
   class Z : public Y {};  // indirectly derived
   typedef X A;
   typedef A B;
   class C : public B {};  // derived from a typedef of X
 
-In the following example, Bar matches isDerivedFrom(hasName("X")):
-  class Foo;
-  typedef Foo X;
-  class Bar : public Foo {};  // derived from a type that X is a typedef of
+  class Foo {};
+  typedef Foo Alias;
+  class Bar : public Alias {};
+  // derived from a type that Alias is a typedef of Foo
+
+
+The matcher cxxRecordDecl(isDerivedFrom(hasName("X")))
+matches Y, Z and C.
+The matcher cxxRecordDecl(isDerivedFrom(hasName("Foo")))
+matches Bar.
 
 In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
   @interface NSObject @end
   @interface Bar : NSObject @end
 
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;
 </pre></td></tr>
 
@@ -7545,38 +10121,90 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Note that a class is not considered to be derived from itself.
 
-Example matches Y, C (Base == hasName("X"))
-  class X;
+Given
+  class X {};
   class Y : public X {};  // directly derived
   class Z : public Y {};  // indirectly derived
   typedef X A;
   typedef A B;
   class C : public B {};  // derived from a typedef of X
 
+The matcher
+cxxRecordDecl(isDirectlyDerivedFrom(namedDecl(hasName("X"))))
+matches Y and C (Base == hasName("X")
+
 In the following example, Bar matches isDerivedFrom(hasName("X")):
-  class Foo;
+  class Foo {};
   typedef Foo X;
   class Bar : public Foo {};  // derived from a type that X is a typedef of
+
+The matcher cxxRecordDecl(isDerivedFrom(hasName("X")))
+matches Bar
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;</td><td class="name" onclick="toggle('isSameOrDerivedFrom0')"><a name="isSameOrDerivedFrom0Anchor">isSameOrDerivedFrom</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt; Base</td></tr>
 <tr><td colspan="4" class="doc" id="isSameOrDerivedFrom0"><pre>Similar to isDerivedFrom(), but also matches classes that directly
 match Base.
+
+Given
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+The matcher
+cxxRecordDecl(isSameOrDerivedFrom(cxxRecordDecl(hasName("X"))),
+isDefinition())
+matches class X {}, class Y : public X {},
+class Z : public Y {} and class C : public B {}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRewrittenBinaryOperator.html">CXXRewrittenBinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasEitherOperand3')"><a name="hasEitherOperand3Anchor">hasEitherOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasEitherOperand3"><pre>Matches if either the left hand side or the right hand side of a
 binary operator or fold expression matches.
+
+Given
+  struct S {};
+  bool operator ==(const S&amp;, const S&amp;);
+
+  void f(int a, const S&amp;lhs, const S&amp;rhs) {
+      a + 0;
+      lhs == rhs;
+      lhs != rhs;
+  }
+
+  template &lt;typename ...Ts&gt;
+  auto sum(Ts... args) {
+    return (0 + ... + args);
+  }
+
+
+The matcher binaryOperator(hasEitherOperand(integerLiteral()))
+matches a + 0.
+The matcher cxxOperatorCallExpr(hasEitherOperand(declRefExpr(to(
+parmVarDecl(hasName("lhs")))))) matches lhs == rhs and
+lhs != rhs.
+The matcher cxxFoldExpr(hasEitherOperand(integerLiteral()))
+matches (0 + ... + args).
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRewrittenBinaryOperator.html">CXXRewrittenBinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasLHS2')"><a name="hasLHS2Anchor">hasLHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLHS2"><pre>Matches the left hand side of binary operator expressions.
 
-Example matches a (matcher = binaryOperator(hasLHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasLHS(expr().bind("lhs")))
+matches a || b,
+with expr()
+matching a.
 </pre></td></tr>
 
 
@@ -7584,39 +10212,59 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasOperands3"><pre>Matches if both matchers match with opposite sides of the binary operator
 or fold expression.
 
-Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
-                                             integerLiteral(equals(2)))
-  1 + 2 // Match
-  2 + 1 // Match
-  1 + 1 // No match
-  2 + 2 // No match
+Given
+void foo() {
+  1 + 2; // Match
+  2 + 1; // Match
+  1 + 1; // No match
+  2 + 2; // No match
+}
+The matcher binaryOperator(hasOperands(integerLiteral(equals(1)),
+                                            integerLiteral(equals(2))))
+matches 1 + 2 and 2 + 1,
+but does not match 1 + 1
+or 2 + 2.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRewrittenBinaryOperator.html">CXXRewrittenBinaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasRHS2')"><a name="hasRHS2Anchor">hasRHS</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasRHS2"><pre>Matches the right hand side of binary operator expressions.
 
-Example matches b (matcher = binaryOperator(hasRHS()))
-  a || b
+Given
+void foo(bool a, bool b) {
+  a || b;
+}
+
+The matcher binaryOperator(hasRHS(expr().bind("rhs")))
+matches a || b,
+with expr()
+matching b.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXTemporaryObjectExpr.html">CXXTemporaryObjectExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc5')"><a name="hasTypeLoc5Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc5"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -7634,17 +10282,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 expression, or an ObjC-message-send expression.
 
 Given
-  void x(int, int, int) { int y; x(1, y, 42); }
-callExpr(hasAnyArgument(declRefExpr()))
-  matches x(1, y, 42)
-with hasAnyArgument(...)
+  void x(int, int, int) { int y = 42; x(1, y, 42); }
+The matcher
+callExpr(hasAnyArgument(ignoringImplicit(declRefExpr()))) matches
+x(1, y, 42) with hasAnyArgument(...)
   matching y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
   void foo(I *i) { [i f:12]; }
+
+The matcher
 objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))
-  matches [i f:12]
+matches [i f:12]
 </pre></td></tr>
 
 
@@ -7652,28 +10302,37 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasArgument2"><pre>Matches the n'th argument of a call expression or a constructor
 call expression.
 
-Example matches y in x(y)
-    (matcher = callExpr(hasArgument(0, declRefExpr())))
+Given
   void x(int) { int y; x(y); }
+The matcher callExpr(hasArgument(0, declRefExpr().bind("arg")))
+matches x(y),
+with declRefExpr() matching y.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXUnresolvedConstructExpr.html">CXXUnresolvedConstructExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc6')"><a name="hasTypeLoc6Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc6"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -7691,19 +10350,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 given matcher; or 2) if the Obj-C message expression's callee's method
 declaration matches the given matcher.
 
-Example matches y.x() (matcher = callExpr(callee(
-                                   cxxMethodDecl(hasName("x")))))
+Example 1
   class Y { public: void x(); };
   void z() { Y y; y.x(); }
 
-Example 2. Matches [I foo] with
-objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))
+The matcher callExpr(callee(cxxMethodDecl(hasName("x"))))
+matches y.x()
 
+Example 2
   @interface I: NSObject
   +(void)foo;
   @end
   ...
   [I foo]
+
+The matcher
+objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))
+matches [I foo]
 </pre></td></tr>
 
 
@@ -7713,12 +10376,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   class Y { void x() { this-&gt;x(); x(); Y y; y.x(); } };
   void f() { f(); }
-callExpr(callee(expr()))
-  matches this-&gt;x(), x(), y.x(), f()
-with callee(...)
-  matching this-&gt;x, x, y.x, f respectively
+
+The matcher callExpr(callee(expr().bind("callee")))
+matches this-&gt;x(), x(), y.x(), f()
+with expr() inside of callee
+matching this-&gt;x, x,
+y.x, f respectively
 
 Given
+  struct Dummy {};
+  // makes sure there is a callee, otherwise there would be no callee,
+  // just a builtin operator
+  Dummy operator+(Dummy, Dummy);
+  // not defining a '*' operator
+
   template &lt;typename... Args&gt;
   auto sum(Args... args) {
       return (0 + ... + args);
@@ -7728,10 +10399,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   auto multiply(Args... args) {
       return (args * ... * 1);
   }
-cxxFoldExpr(callee(expr()))
-  matches (args * ... * 1)
-with callee(...)
-  matching *
+
+The matcher cxxFoldExpr(callee(expr().bind("op")))
+matches (0 + ... + args)
+with callee(...) matching *,
+but does not match (args * ... * 1).
+A CXXFoldExpr only has an UnresolvedLookupExpr as a callee.
+When there are no define operators that could be used instead of builtin
+ones, then there will be no callee .
 
 Note: Callee cannot take the more general internal::Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;
 because this introduces ambiguous overloads with calls to Callee taking a
@@ -7746,8 +10421,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   void f(int i);
   int y;
-  f(y);
-callExpr(
+  void foo() {
+    f(y);
+  }
+The matcher callExpr(
   forEachArgumentWithParam(
     declRefExpr(to(varDecl(hasName("y")))),
     parmVarDecl(hasType(isInteger()))
@@ -7770,14 +10447,15 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   void f(int i);
-  int y;
-  f(y);
-  void (*f_ptr)(int) = f;
-  f_ptr(y);
-callExpr(
+  void foo(int y) {
+    f(y);
+    void (*f_ptr)(int) = f;
+    f_ptr(y);
+  }
+The matcher callExpr(
   forEachArgumentWithParamType(
     declRefExpr(to(varDecl(hasName("y")))),
-    qualType(isInteger()).bind("type)
+    qualType(isInteger()).bind("type")
 ))
   matches f(y) and f_ptr(y)
 with declRefExpr(...)
@@ -7792,17 +10470,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 expression, or an ObjC-message-send expression.
 
 Given
-  void x(int, int, int) { int y; x(1, y, 42); }
-callExpr(hasAnyArgument(declRefExpr()))
-  matches x(1, y, 42)
-with hasAnyArgument(...)
+  void x(int, int, int) { int y = 42; x(1, y, 42); }
+The matcher
+callExpr(hasAnyArgument(ignoringImplicit(declRefExpr()))) matches
+x(1, y, 42) with hasAnyArgument(...)
   matching y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
   void foo(I *i) { [i f:12]; }
+
+The matcher
 objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))
-  matches [i f:12]
+matches [i f:12]
 </pre></td></tr>
 
 
@@ -7810,15 +10490,17 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasArgument0"><pre>Matches the n'th argument of a call expression or a constructor
 call expression.
 
-Example matches y in x(y)
-    (matcher = callExpr(hasArgument(0, declRefExpr())))
+Given
   void x(int) { int y; x(y); }
+The matcher callExpr(hasArgument(0, declRefExpr().bind("arg")))
+matches x(y),
+with declRefExpr() matching y.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration14')"><a name="hasDeclaration14Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration14"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -7828,17 +10510,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -7855,9 +10545,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 extension, matches the constant given in the statement.
 
 Given
-  switch (1) { case 1: case 1+1: case 3 ... 4: ; }
-caseStmt(hasCaseConstant(integerLiteral()))
-  matches "case 1:"
+  void foo() {
+    switch (1) { case 1: break; case 1+1: break; case 3 ... 4: break; }
+  }
+The matcher
+caseStmt(hasCaseConstant(constantExpr(has(integerLiteral()))))
+matches case 1: break.
 </pre></td></tr>
 
 
@@ -7865,14 +10558,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasSourceExpression0"><pre>Matches if the cast's source expression
 or opaque value's source expression matches the given matcher.
 
-Example 1: matches "a string"
-(matcher = castExpr(hasSourceExpression(cxxConstructExpr())))
-class URL { URL(string); };
-URL url = "a string";
+Given
+ struct URL { URL(const char*); };
+ URL url = "a string";
+
+The matcher castExpr(hasSourceExpression(cxxConstructExpr()))
+matches "a string".
+
+Given
+void foo(bool b) {
+  int a = b ?: 1;
+}
 
-Example 2: matches 'b' (matcher =
-opaqueValueExpr(hasSourceExpression(implicitCastExpr(declRefExpr())))
-int a = b ?: 1;
+The matcher
+opaqueValueExpr(hasSourceExpression(
+              implicitCastExpr(has(
+                implicitCastExpr(has(declRefExpr()))))))
+matches b twice, for the conditiona and the true expression.
 </pre></td></tr>
 
 
@@ -7892,13 +10594,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template &lt;typename T, typename U&gt;
   void f(T&amp;&amp; t, U&amp;&amp; u) {}
 
-  bool B = false;
-  f(R, B);
-templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-  matches twice, with expr() matching 'R * 2' and 'R * 4'
-functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
-  and 'bool'
+  void foo() {
+    bool B = false;
+    f(R, B);
+  }
+
+The matcher
+templateSpecializationType(forEachTemplateArgument(isExpr(expr().bind("t_arg"))))
+matches Matrix&lt;int, R * 2, R * 4&gt; twice, with
+expr() matching R * 2 and
+R * 4.
+The matcher
+functionDecl(forEachTemplateArgument(refersToType(qualType().bind("type"))))
+matches the specialization of f twice,
+with qualType() matching
+unsigned and
+bool.
 </pre></td></tr>
 
 
@@ -7911,9 +10622,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;typename T&gt; class A {};
   A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+hasTypeLoc(loc(asString("int"))))))))) matches A&lt;int&gt; a.
 </pre></td></tr>
 
 
@@ -7927,15 +10640,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;&gt; class A&lt;double&gt; {};
   A&lt;int&gt; a;
 
-  template&lt;typename T&gt; f() {};
+  template&lt;typename T&gt; void f() {};
   void func() { f&lt;int&gt;(); };
 
-classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToType(asString("int"))))
-  matches the specialization A&lt;int&gt;
 
-functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher classTemplateSpecializationDecl(
+                        hasAnyTemplateArgument(
+                          refersToType(asString("int"))))
+matches class A&lt;int&gt;.
+
+The matcher
+functionDecl(hasAnyTemplateArgument(
+              refersToType(asString("int"))))
+matches the instantiation of f.
 </pre></td></tr>
 
 
@@ -7943,11 +10660,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasSpecializedTemplate0"><pre>Matches the specialized template of a specialization declaration.
 
 Given
-  template&lt;typename T&gt; class A {}; #1
-  template&lt;&gt; class A&lt;int&gt; {}; #2
-classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl()))
-  matches '#2' with classTemplateDecl() matching the class template
-  declaration of 'A' at #1.
+  template&lt;typename T&gt; class A {}; // #1
+  template&lt;&gt; class A&lt;int&gt; {}; // #2
+
+The matcher
+classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl().bind("ctd")))
+matches template&lt;&gt; class A&lt;int&gt; {},
+with classTemplateDecl() matching the class template
+declaration template &lt;typename T&gt; class A {}.
 </pre></td></tr>
 
 
@@ -7960,9 +10680,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T, typename U&gt; class A {};
   A&lt;double, int&gt; b;
   A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+hasTypeLoc(loc(asString("double")))))))))
+matches A&lt;double, int&gt; b, but not double&gt; c}.
 </pre></td></tr>
 
 
@@ -7973,17 +10696,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   template&lt;typename T, typename U&gt; class A {};
-  A&lt;bool, int&gt; b;
-  A&lt;int, bool&gt; c;
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
 
   template&lt;typename T&gt; void f() {}
   void func() { f&lt;int&gt;(); };
+
+The matcher
 classTemplateSpecializationDecl(hasTemplateArgument(
     1, refersToType(asString("int"))))
-  matches the specialization A&lt;bool, int&gt;
+matches the specialization class A&lt;double, int&gt;.
 
-functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher functionDecl(hasTemplateArgument(0,
+                        refersToType(asString("int"))))
+matches the specialization of f.
 </pre></td></tr>
 
 
@@ -7995,8 +10721,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   struct A {};
   A a[7];
   int b[7];
-arrayType(hasElementType(builtinType()))
-  matches "int b[7]"
+
+
+The matcher arrayType(hasElementType(builtinType()))
+int[7]
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ArrayType.html">ArrayType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ComplexType.html">ComplexType</a>&gt;
 </pre></td></tr>
@@ -8005,19 +10733,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CompoundLiteralExpr.html">CompoundLiteralExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc7')"><a name="hasTypeLoc7Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc7"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -8035,11 +10770,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 a given matcher. Also matches StmtExprs that have CompoundStmt as children.
 
 Given
-  { {}; 1+2; }
-hasAnySubstatement(compoundStmt())
-  matches '{ {}; 1+2; }'
+void foo() { { {}; 1+2; } }
+The matcher
+compoundStmt(hasAnySubstatement(compoundStmt().bind("compound")))
+{ {}; 1+2; } and { { {}; 1+2; } }
 with compoundStmt()
-  matching '{}'
+matching {} and { {}; 1+2; }.
 </pre></td></tr>
 
 
@@ -8050,25 +10786,35 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecayedType.html">DecayedType</a>&gt;</td><td class="name" onclick="toggle('hasDecayedType0')"><a name="hasDecayedType0Anchor">hasDecayedType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerType</td></tr>
 <tr><td colspan="4" class="doc" id="hasDecayedType0"><pre>Matches the decayed type, whoes decayed type matches InnerMatcher
+
+Given
+  void f(int i[]) {
+    i[1] = 0;
+  }
+
+The matcher parmVarDecl(hasType(decayedType()))
+matches int i[].
 </pre></td></tr>
 
 
@@ -8081,15 +10827,17 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;typename T&gt; class A {};
   A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+hasTypeLoc(loc(asString("int"))))))))) matches A&lt;int&gt; a.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration11')"><a name="hasDeclaration11Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration11"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -8099,17 +10847,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -8130,9 +10886,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T, typename U&gt; class A {};
   A&lt;double, int&gt; b;
   A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+hasTypeLoc(loc(asString("double")))))))))
+matches A&lt;double, int&gt; b, but not double&gt; c}.
 </pre></td></tr>
 
 
@@ -8140,18 +10899,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="throughUsingDecl0"><pre>Matches if a node refers to a declaration through a specific
 using shadow declaration.
 
-Examples:
+Given
   namespace a { int f(); }
   using a::f;
   int x = f();
-declRefExpr(throughUsingDecl(anything()))
-  matches f
+
+The matcher declRefExpr(throughUsingDecl(anything()))
+matches f
 
   namespace a { class X{}; }
   using a::X;
   X x;
-typeLoc(loc(usingType(throughUsingDecl(anything()))))
-  matches X
+
+The matcher typeLoc(loc(usingType(throughUsingDecl(anything()))))
+matches X
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingType.html">UsingType</a>&gt;
 </pre></td></tr>
@@ -8161,10 +10922,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="to0"><pre>Matches a DeclRefExpr that refers to a declaration that matches the
 specified matcher.
 
-Example matches x in if(x)
-    (matcher = declRefExpr(to(varDecl(hasName("x")))))
-  bool x;
-  if (x) {}
+Given
+  void foo() {
+    bool x;
+    if (x) {}
+  }
+
+The matcher declRefExpr(to(varDecl(hasName("x"))))
+matches x inside the condition of the if-stmt.
 </pre></td></tr>
 
 
@@ -8174,16 +10939,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Note that this does not work for global declarations because the AST
 breaks up multiple-declaration DeclStmt's into multiple single-declaration
 DeclStmt's.
-Example: Given non-global declarations
-  int a, b = 0;
-  int c;
-  int d = 2, e;
-declStmt(containsDeclaration(
+
+Given non-global declarations
+  void foo() {
+    int a, b = 0;
+    int c;
+    int d = 2, e;
+  }
+The matcher declStmt(containsDeclaration(
       0, varDecl(hasInitializer(anything()))))
-  matches only 'int d = 2, e;', and
-declStmt(containsDeclaration(1, varDecl()))
-  matches 'int a, b = 0' as well as 'int d = 2, e;'
-  but 'int c;' is not matched.
+matches int d = 2, e;.
+The matcher declStmt(containsDeclaration(1, varDecl()))
+matches int a, b = 0; and int d = 2, e;
+but does not match int c;.
 </pre></td></tr>
 
 
@@ -8191,29 +10959,39 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasSingleDecl0"><pre>Matches the Decl of a DeclStmt which has a single declaration.
 
 Given
-  int a, b;
-  int c;
-declStmt(hasSingleDecl(anything()))
-  matches 'int c;' but not 'int a, b;'.
+  void foo() {
+    int a, b;
+    int c;
+  }
+The matcher declStmt(hasSingleDecl(anything()))
+matches int c;
+but does not match int a, b;
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclaratorDecl.html">DeclaratorDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc8')"><a name="hasTypeLoc8Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc8"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -8237,8 +11015,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     }
   }
 
-cxxRcordDecl(hasDeclContext(namedDecl(hasName("M")))) matches the
-declaration of class D.
+
+The matcher cxxRecordDecl(hasDeclContext(namedDecl(hasName("M"))))
+  matches the declaration of D.
 </pre></td></tr>
 
 
@@ -8248,8 +11027,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   decltype(1) a = 1;
   decltype(2.0) b = 2.0;
-decltypeType(hasUnderlyingType(isInteger()))
-  matches the type of "a"
+
+
+The matcher decltypeType(hasUnderlyingType(isInteger()))
+matches the type decltype(1) of the variable
+declaration of a .
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecltypeType.html">DecltypeType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingType.html">UsingType</a>&gt;
 </pre></td></tr>
@@ -8266,16 +11048,17 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
     f = 42;
 }
-The matcher:
-  decompositionDecl(hasAnyBinding(bindingDecl(hasName("f").bind("fBinding"))))
-matches the decomposition decl with 'f' bound to "fBinding".
+
+The matcher
+ decompositionDecl(hasAnyBinding(bindingDecl(hasName("f")).bind("fBinding")))
+matches auto &amp;[f, s, t] = arr with 'f' bound to "fBinding".
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecompositionDecl.html">DecompositionDecl</a>&gt;</td><td class="name" onclick="toggle('hasBinding0')"><a name="hasBinding0Anchor">hasBinding</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BindingDecl.html">BindingDecl</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasBinding0"><pre>Matches the Nth binding of a DecompositionDecl.
 
-For example, in:
+Given
 void foo()
 {
     int arr[3];
@@ -8283,10 +11066,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
     f = 42;
 }
-The matcher:
-  decompositionDecl(hasBinding(0,
-  bindingDecl(hasName("f").bind("fBinding"))))
-matches the decomposition decl with 'f' bound to "fBinding".
+
+The matcher decompositionDecl(hasBinding(0,
+  bindingDecl(hasName("f")).bind("fBinding")))
+matches auto &amp;[f, s, t] = arr with 'f' bound to "fBinding".
 </pre></td></tr>
 
 
@@ -8297,20 +11080,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
@@ -8318,8 +11103,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition3"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 
@@ -8334,14 +11124,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
   class D {};
   class D d;
-elaboratedTypeLoc(hasNamedTypeLoc(templateSpecializationTypeLoc()));
-  matches the `TypeLoc` of the variable declaration of `c`, but not `d`.
+
+The matcher
+elaboratedTypeLoc(hasNamedTypeLoc(templateSpecializationTypeLoc()))
+  matches class C&lt;int&gt;, but not D}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ElaboratedType.html">ElaboratedType</a>&gt;</td><td class="name" onclick="toggle('hasQualifier0')"><a name="hasQualifier0Anchor">hasQualifier</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifier.html">NestedNameSpecifier</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasQualifier0"><pre>Matches ElaboratedTypes whose qualifier, a NestedNameSpecifier,
-matches InnerMatcher if the qualifier exists.
+  matches InnerMatcher if the qualifier exists.
 
 Given
   namespace N {
@@ -8351,8 +11143,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   }
   N::M::D d;
 
-elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N"))))
-matches the type of the variable declaration of d.
+
+The matcher
+elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))
+  matches the type N::M::D of the variable declaration
+  of d.
 </pre></td></tr>
 
 
@@ -8362,20 +11157,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   namespace N {
     namespace M {
-      class D {};
+      enum E { Ok };
     }
   }
-  N::M::D d;
+  N::M::E e = N::M::Ok;
 
-elaboratedType(namesType(recordType(
-hasDeclaration(namedDecl(hasName("D")))))) matches the type of the variable
-declaration of d.
+
+The matcher elaboratedType(namesType(enumType()))
+matches the type N::M::E of the declaration of e .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1EnumType.html">EnumType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration10')"><a name="hasDeclaration10Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration10"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -8385,17 +11180,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -8412,25 +11215,37 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 (Note: Clang's AST refers to other conversions as "casts" too, and calls
 actual casts "explicit" casts.)
+
+  unsigned int a = (unsigned int)0;
+
+The matcher explicitCastExpr(hasDestinationType(
+qualType(isUnsignedInteger()))) matches (unsigned int)0.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ExplicitCastExpr.html">ExplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc9')"><a name="hasTypeLoc9Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc9"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -8453,21 +11268,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
 declaration of x.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              cxxRecordDecl(hasName("X"))))
  class X {};
  void y(X &amp;x) { x; X z; }
  class Y { friend class X; };
  class Z : public virtual X {};
 
-Example matches class Derived
-(matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z.
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X.
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches
+class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
+
+Given
 class Base {};
 class Derived : Base {};
 
+The matcher
+cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base")))))
+matches class Derived : Base {}.
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FriendDecl.html">FriendDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;,
 Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -8477,17 +11302,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasType0"><pre>Matches if the expression's or declaration's type matches a type
 matcher.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and U (matcher = typedefDecl(hasType(asString("int")))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              asString("class X")))
+Exmaple
  class X {};
  void y(X &amp;x) { x; X z; }
  typedef int U;
  class Y { friend class X; };
  class Z : public virtual X {};
+
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z
+The matcher typedefDecl(hasType(asString("int")))
+matches typedef int U
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
 </pre></td></tr>
 
 
@@ -8504,15 +11337,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 appear in the C++17 AST.
 
 Given
-
 struct H {};
 H G();
 void f() {
   H D = G();
 }
 
-``varDecl(hasInitializer(ignoringElidableConstructorCall(callExpr())))``
-matches ``H D = G()`` in C++11 through C++17 (and beyond).
+
+The matcher
+varDecl(hasInitializer(ignoringElidableConstructorCall(callExpr())))
+matches H D = G().
 </pre></td></tr>
 
 
@@ -8523,19 +11357,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Parentheses and explicit casts are not discarded.
 Given
   int arr[5];
-  int a = 0;
+  const int a = 0;
   char b = 0;
   const int c = a;
   int *d = arr;
   long e = (long) 0l;
-The matchers
-   varDecl(hasInitializer(ignoringImpCasts(integerLiteral())))
-   varDecl(hasInitializer(ignoringImpCasts(declRefExpr())))
-would match the declarations for a, b, c, and d, but not e.
-While
-   varDecl(hasInitializer(integerLiteral()))
-   varDecl(hasInitializer(declRefExpr()))
-only match the declarations for a.
+The matcher
+varDecl(hasInitializer(ignoringImpCasts(integerLiteral())))
+matches a and b,
+but does not match e.
+The matcher
+varDecl(hasInitializer(ignoringImpCasts(declRefExpr())))
+matches c and d.
+
+The matcher
+varDecl(hasInitializer(integerLiteral()))
+matches a,
+but does not match b or e.
+The matcher varDecl(hasInitializer(declRefExpr()))
+does not match c or d.
 </pre></td></tr>
 
 
@@ -8544,17 +11384,34 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 nodes are stripped off.
 
 Parentheses and explicit casts are not discarded.
+
 Given
-  class C {};
-  C a = C();
-  C b;
-  C c = b;
-The matchers
-   varDecl(hasInitializer(ignoringImplicit(cxxConstructExpr())))
-would match the declarations for a, b, and c.
-While
-   varDecl(hasInitializer(cxxConstructExpr()))
-only match the declarations for b and c.
+  void f(int param) {
+    int a = 0;
+    int b = param;
+    const int c = 0;
+    const int d = param;
+    int e = (0U);
+    int f = (int)0.0;
+    const int g = ((int)(((0))));
+  }
+
+The matcher
+varDecl(hasInitializer(ignoringImplicit(integerLiteral())))
+matches int a = 0 and const int c = 0,
+but not int e = (0U) and ((int)(((0))).
+The matcher
+varDecl(hasInitializer(integerLiteral()))
+matches int a = 0 and const int c = 0,
+but not int e = (0U) and ((int)(((0))).
+
+The matcher
+varDecl(hasInitializer(ignoringImplicit(declRefExpr())))
+matches int b = param and const int d = param.
+The matcher
+varDecl(hasInitializer(declRefExpr()))
+matches neither int b = param nor const int d = param,
+because an l-to-r-value cast happens.
 </pre></td></tr>
 
 
@@ -8568,12 +11425,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   char b = (0);
   void* c = reinterpret_cast&lt;char*&gt;(0);
   char d = char(0);
+
 The matcher
-   varDecl(hasInitializer(ignoringParenCasts(integerLiteral())))
-would match the declarations for a, b, c, and d.
-while
-   varDecl(hasInitializer(integerLiteral()))
-only match the declaration for a.
+varDecl(hasInitializer(ignoringParenCasts(integerLiteral())))
+matches a, b, c
+and d.
+The matcher
+varDecl(hasInitializer(integerLiteral()))
+matches a.
 </pre></td></tr>
 
 
@@ -8589,14 +11448,21 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   const int c = a;
   int *d = (arr);
   long e = ((long) 0l);
-The matchers
-   varDecl(hasInitializer(ignoringParenImpCasts(integerLiteral())))
-   varDecl(hasInitializer(ignoringParenImpCasts(declRefExpr())))
-would match the declarations for a, b, c, and d, but not e.
-while
-   varDecl(hasInitializer(integerLiteral()))
-   varDecl(hasInitializer(declRefExpr()))
-would only match the declaration for a.
+
+The matcher
+varDecl(hasInitializer(ignoringParenImpCasts(integerLiteral())))
+matches a and b,
+but does not match e.
+The matcher
+varDecl(hasInitializer(ignoringParenImpCasts(declRefExpr())))
+matches c and d.
+
+The matcher
+varDecl(hasInitializer(integerLiteral()))
+matches a,
+but does not match b or e.
+The matcher varDecl(hasInitializer(declRefExpr()))
+does not match c, or d.
 </pre></td></tr>
 
 
@@ -8606,8 +11472,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   const char* str = ("my-string");
 The matcher
-  implicitCastExpr(hasSourceExpression(ignoringParens(stringLiteral())))
-would match the implicit cast resulting from the assignment.
+implicitCastExpr(hasSourceExpression(ignoringParens(stringLiteral())))
+would match the implicit cast resulting from the assignment
+("my-string").
 </pre></td></tr>
 
 
@@ -8620,10 +11487,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     int b = 3;
     int c;
   };
+
+The matcher
 fieldDecl(hasInClassInitializer(integerLiteral(equals(2))))
-  matches 'int a;' but not 'int b;'.
-fieldDecl(hasInClassInitializer(anything()))
-  matches 'int a;' and 'int b;' but not 'int c;'.
+matches a,
+but does not match b.
+The matcher fieldDecl(hasInClassInitializer(anything()))
+matches a and b,
+but does not match c.
 </pre></td></tr>
 
 
@@ -8634,20 +11505,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
@@ -8655,28 +11528,38 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition1"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ForStmt.html">ForStmt</a>&gt;</td><td class="name" onclick="toggle('hasIncrement0')"><a name="hasIncrement0Anchor">hasIncrement</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasIncrement0"><pre>Matches the increment statement of a for loop.
 
-Example:
-    forStmt(hasIncrement(unaryOperator(hasOperatorName("++"))))
-matches '++x' in
-    for (x; x &lt; N; ++x) { }
+Given
+void foo(int N) {
+    for (int x = 0; x &lt; N; ++x) { }
+}
+The matcher
+forStmt(hasIncrement(unaryOperator(hasOperatorName("++"))))
+matches for (int x = 0; x &lt; N; ++x) { }
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ForStmt.html">ForStmt</a>&gt;</td><td class="name" onclick="toggle('hasLoopInit0')"><a name="hasLoopInit0Anchor">hasLoopInit</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasLoopInit0"><pre>Matches the initialization statement of a for loop.
 
-Example:
-    forStmt(hasLoopInit(declStmt()))
-matches 'int x = 0' in
+Given
+void foo(int N) {
     for (int x = 0; x &lt; N; ++x) { }
+}
+The matcher forStmt(hasLoopInit(declStmt()))
+matches for (int x = 0; x &lt; N; ++x) { }
 </pre></td></tr>
 
 
@@ -8690,21 +11573,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
 declaration of x.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              cxxRecordDecl(hasName("X"))))
  class X {};
  void y(X &amp;x) { x; X z; }
  class Y { friend class X; };
  class Z : public virtual X {};
 
-Example matches class Derived
-(matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z.
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X.
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches
+class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
+
+Given
 class Base {};
 class Derived : Base {};
 
+The matcher
+cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base")))))
+matches class Derived : Base {}.
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FriendDecl.html">FriendDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;,
 Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -8714,17 +11607,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasType1"><pre>Matches if the expression's or declaration's type matches a type
 matcher.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and U (matcher = typedefDecl(hasType(asString("int")))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              asString("class X")))
+Exmaple
  class X {};
  void y(X &amp;x) { x; X z; }
  typedef int U;
  class Y { friend class X; };
  class Z : public virtual X {};
+
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z
+The matcher typedefDecl(hasType(asString("int")))
+matches typedef int U
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
 </pre></td></tr>
 
 
@@ -8744,13 +11645,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template &lt;typename T, typename U&gt;
   void f(T&amp;&amp; t, U&amp;&amp; u) {}
 
-  bool B = false;
-  f(R, B);
-templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-  matches twice, with expr() matching 'R * 2' and 'R * 4'
-functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
-  and 'bool'
+  void foo() {
+    bool B = false;
+    f(R, B);
+  }
+
+The matcher
+templateSpecializationType(forEachTemplateArgument(isExpr(expr().bind("t_arg"))))
+matches Matrix&lt;int, R * 2, R * 4&gt; twice, with
+expr() matching R * 2 and
+R * 4.
+The matcher
+functionDecl(forEachTemplateArgument(refersToType(qualType().bind("type"))))
+matches the specialization of f twice,
+with qualType() matching
+unsigned and
+bool.
 </pre></td></tr>
 
 
@@ -8763,12 +11673,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   void f();
   void f() {}
   void g();
-functionDecl(hasAnyBody(compoundStmt()))
-  matches both 'void f();'
-  and 'void f() {}'
+The matcher functionDecl(hasAnyBody(compoundStmt()))
+  matches f
+  and f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void g();'
+  matching {}
+  but does not match void g();
 </pre></td></tr>
 
 
@@ -8780,23 +11690,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x, int y, int z) {} };
-cxxMethodDecl(hasAnyParameter(hasName("y")))
-  matches f(int x, int y, int z) {}
+
+The matcher cxxMethodDecl(hasAnyParameter(hasName("y")))
+  matches f
 with hasAnyParameter(...)
   matching int y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
+
 the matcher objcMethodDecl(hasAnyParameter(hasName("y")))
-matches the declaration of method f with hasParameter
+  matches the declaration of method f with hasParameter
 matching y.
 
 For blocks, given
   b = ^(int y) { printf("%d", y) };
 
+
 the matcher blockDecl(hasAnyParameter(hasName("y")))
-matches the declaration of the block b with hasParameter
+  matches the declaration of the block b with hasParameter
 matching y.
 </pre></td></tr>
 
@@ -8810,9 +11723,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;typename T&gt; class A {};
   A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+hasTypeLoc(loc(asString("int"))))))))) matches A&lt;int&gt; a.
 </pre></td></tr>
 
 
@@ -8826,15 +11741,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;&gt; class A&lt;double&gt; {};
   A&lt;int&gt; a;
 
-  template&lt;typename T&gt; f() {};
+  template&lt;typename T&gt; void f() {};
   void func() { f&lt;int&gt;(); };
 
-classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToType(asString("int"))))
-  matches the specialization A&lt;int&gt;
 
-functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher classTemplateSpecializationDecl(
+                        hasAnyTemplateArgument(
+                          refersToType(asString("int"))))
+matches class A&lt;int&gt;.
+
+The matcher
+functionDecl(hasAnyTemplateArgument(
+              refersToType(asString("int"))))
+matches the instantiation of f.
 </pre></td></tr>
 
 
@@ -8845,20 +11764,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
@@ -8873,15 +11794,27 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     explicit S(double); // #2
     operator int(); // #3
     explicit operator bool(); // #4
-    explicit(false) S(bool) // # 7
-    explicit(true) S(char) // # 8
-    explicit(b) S(S) // # 9
+    explicit(false) S(bool); // # 7
+    explicit(true) S(char); // # 8
+    explicit(b) S(float); // # 9
   };
-  S(int) -&gt; S&lt;true&gt; // #5
-  explicit S(double) -&gt; S&lt;false&gt; // #6
-cxxConstructorDecl(hasExplicitSpecifier(constantExpr())) will match #7, #8 and #9, but not #1 or #2.
-cxxConversionDecl(hasExplicitSpecifier(constantExpr())) will not match #3 or #4.
-cxxDeductionGuideDecl(hasExplicitSpecifier(constantExpr())) will not match #5 or #6.
+  S(int) -&gt; S&lt;true&gt;; // #5
+  explicit S(double) -&gt; S&lt;false&gt;; // #6
+
+The matcher
+cxxConstructorDecl(hasExplicitSpecifier(constantExpr())) matches
+explicit(false) S(bool) and explicit(true) S(char),
+but does not match explicit(b) S(float), S(int) or
+explicit S(double).
+The matcher
+cxxConversionDecl(hasExplicitSpecifier(constantExpr())) does not
+match operator int() or explicit operator bool().
+Matcher
+The matcher
+cxxDeductionGuideDecl(hasExplicitSpecifier(declRefExpr()))
+matches the implicitly generated deduction guide
+auto (float) -&gt; S&lt;b&gt; of the constructor
+S(float)}.
 </pre></td></tr>
 
 
@@ -8891,15 +11824,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x) {} };
-cxxMethodDecl(hasParameter(0, hasType(varDecl())))
-  matches f(int x) {}
+
+The matcher
+cxxMethodDecl(hasParameter(0, hasType(asString("int"))))
+matches f
 with hasParameter(...)
-  matching int x
+matching int x.
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
-the matcher objcMethodDecl(hasParameter(0, hasName("y")))
+
+The matcher objcMethodDecl(hasParameter(0, hasName("y")))
 matches the declaration of method f with hasParameter
 matching y.
 </pre></td></tr>
@@ -8911,9 +11847,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   int f() { return 5; }
   void g() {}
-functionDecl(hasReturnTypeLoc(loc(asString("int"))))
-  matches the declaration of `f`, but not `g`.
-</pre></td></tr>
+The matcher functionDecl(hasReturnTypeLoc(loc(asString("int"))))
+  matches the declaration of f, but not
+  </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('hasTemplateArgumentLoc2')"><a name="hasTemplateArgumentLoc2Anchor">hasTemplateArgumentLoc</a></td><td>unsigned Index, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt; InnerMatcher</td></tr>
@@ -8925,9 +11861,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T, typename U&gt; class A {};
   A&lt;double, int&gt; b;
   A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+hasTypeLoc(loc(asString("double")))))))))
+matches A&lt;double, int&gt; b, but not double&gt; c}.
 </pre></td></tr>
 
 
@@ -8938,27 +11877,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   template&lt;typename T, typename U&gt; class A {};
-  A&lt;bool, int&gt; b;
-  A&lt;int, bool&gt; c;
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
 
   template&lt;typename T&gt; void f() {}
   void func() { f&lt;int&gt;(); };
+
+The matcher
 classTemplateSpecializationDecl(hasTemplateArgument(
     1, refersToType(asString("int"))))
-  matches the specialization A&lt;bool, int&gt;
+matches the specialization class A&lt;double, int&gt;.
 
-functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher functionDecl(hasTemplateArgument(0,
+                        refersToType(asString("int"))))
+matches the specialization of f.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FunctionDecl.html">FunctionDecl</a>&gt;</td><td class="name" onclick="toggle('returns0')"><a name="returns0Anchor">returns</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="returns0"><pre>Matches the return type of a function declaration.
 
-Given:
+Given
   class X { int f() { return 1; } };
-cxxMethodDecl(returns(asString("int")))
-  matches int f() { return 1; }
+
+The matcher cxxMethodDecl(returns(asString("int")))
+  matches f
 </pre></td></tr>
 
 
@@ -8966,8 +11909,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition0"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 
@@ -8975,25 +11923,37 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasConditionVariableStatement0"><pre>Matches the condition variable statement in an if statement.
 
 Given
+struct A {};
+A* GetAPointer();
+void foo() {
   if (A* a = GetAPointer()) {}
-hasConditionVariableStatement(...)
-  matches 'A* a = GetAPointer()'.
+}
+
+The matcher ifStmt(hasConditionVariableStatement(declStmt()))
+if (A* a = GetAPointer()) {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IfStmt.html">IfStmt</a>&gt;</td><td class="name" onclick="toggle('hasElse0')"><a name="hasElse0Anchor">hasElse</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasElse0"><pre>Matches the else-statement of an if statement.
 
-Examples matches the if statement
-  (matcher = ifStmt(hasElse(cxxBoolLiteral(equals(true)))))
+Given
+void foo() {
   if (false) false; else true;
+}
+
+The matcher ifStmt(hasElse(cxxBoolLiteral(equals(true))))
+if (false) false; else true
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IfStmt.html">IfStmt</a>&gt;</td><td class="name" onclick="toggle('hasInitStatement0')"><a name="hasInitStatement0Anchor">hasInitStatement</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasInitStatement0"><pre>Matches selection statements with initializer.
 
-Given:
+Given
+ struct vec { int* begin(); int* end(); };
+ int foobar();
+ vec&amp; get_range();
  void foo() {
    if (int i = foobar(); i &gt; 0) {}
    switch (int i = foobar(); i) {}
@@ -9004,48 +11964,77 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
    switch (foobar()) {}
    for (auto&amp; x : get_range()) {}
  }
-ifStmt(hasInitStatement(anything()))
-  matches the if statement in foo but not in bar.
-switchStmt(hasInitStatement(anything()))
-  matches the switch statement in foo but not in bar.
-cxxForRangeStmt(hasInitStatement(anything()))
-  matches the range for statement in foo but not in bar.
+
+The matcher ifStmt(hasInitStatement(anything()))
+  matches the if statement if (int i = foobar(); i &gt; 0) {}
+  in foo but not if (foobar() &gt; 0) {} in bar.
+The matcher switchStmt(hasInitStatement(anything()))
+  matches the switch statement switch (int i = foobar(); i) {}
+  in foo but not switch (foobar()) {} in bar.
+The matcher cxxForRangeStmt(hasInitStatement(anything()))
+  matches the range for statement
+  for (auto&amp; a = get_range(); auto&amp; x : a) {} in foo
+  but not for (auto&amp; x : get_range()) {} in bar.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1IfStmt.html">IfStmt</a>&gt;</td><td class="name" onclick="toggle('hasThen0')"><a name="hasThen0Anchor">hasThen</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasThen0"><pre>Matches the then-statement of an if statement.
 
-Examples matches the if statement
-  (matcher = ifStmt(hasThen(cxxBoolLiteral(equals(true)))))
+Given
+void foo() {
   if (false) true; else false;
+}
+
+The matcher ifStmt(hasThen(cxxBoolLiteral(equals(true))))
+if (false) true; else false
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ImplicitCastExpr.html">ImplicitCastExpr</a>&gt;</td><td class="name" onclick="toggle('hasImplicitDestinationType0')"><a name="hasImplicitDestinationType0Anchor">hasImplicitDestinationType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasImplicitDestinationType0"><pre>Matches implicit casts whose destination type matches a given
 matcher.
+
+Given
+  unsigned int a = 0;
+
+The matcher
+implicitCastExpr(hasImplicitDestinationType(
+qualType(isUnsignedInteger()))) matches 0.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1InitListExpr.html">InitListExpr</a>&gt;</td><td class="name" onclick="toggle('hasInit0')"><a name="hasInit0Anchor">hasInit</a></td><td>unsigned N, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasInit0"><pre>Matches the n'th item of an initializer list expression.
 
-Example matches y.
-    (matcher = initListExpr(hasInit(0, expr())))
-  int x{y}.
+Given
+  int y = 42;
+  int x{y};
+
+The matcher initListExpr(hasInit(0, expr()))
+matches {y}.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1InitListExpr.html">InitListExpr</a>&gt;</td><td class="name" onclick="toggle('hasSyntacticForm0')"><a name="hasSyntacticForm0Anchor">hasSyntacticForm</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasSyntacticForm0"><pre>Matches the syntactic form of init list expressions
 (if expression have it).
+
+Given
+  int a[] = { 1, 2 };
+  struct B { int x, y; };
+  struct B b = { 5, 6 };
+
+
+The matcher
+initListExpr(hasSyntacticForm(expr().bind("syntactic")))
+matches { 1, 2 } and { 5, 6 }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1InjectedClassNameType.html">InjectedClassNameType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration9')"><a name="hasDeclaration9Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration9"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9055,17 +12044,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9079,7 +12076,7 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1LabelStmt.html">LabelStmt</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration8')"><a name="hasDeclaration8Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration8"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9089,17 +12086,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9122,9 +12127,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     auto f = [x](){};
     auto g = [x = 1](){};
   }
-In the matcher
-lambdaExpr(hasAnyCapture(lambdaCapture(capturesVar(hasName("x")))),
-capturesVar(hasName("x")) matches `x` and `x = 1`.
+
+The matcher
+lambdaExpr(hasAnyCapture(
+    lambdaCapture(capturesVar(hasName("x"))).bind("capture")))
+matches [x](){} and [x = 1](){}, with
+lambdaCapture(capturesVar(hasName("x"))).bind("capture")
+matching x and x = 1.
 </pre></td></tr>
 
 
@@ -9133,13 +12142,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int main() {
-    int x, y;
+    int x;
+    int y;
     float z;
     auto f = [=]() { return x + y + z; };
   }
-lambdaExpr(forEachLambdaCapture(
-    lambdaCapture(capturesVar(varDecl(hasType(isInteger()))))))
-will trigger two matches, binding for 'x' and 'y' respectively.
+
+The matcher lambdaExpr(forEachLambdaCapture(
+    lambdaCapture(capturesVar(
+    varDecl(hasType(isInteger())).bind("captured")))))
+matches [=]() { return x + y + z; } two times,
+with varDecl(hasType(isInteger())) matching
+int x and int y.
 </pre></td></tr>
 
 
@@ -9151,15 +12165,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     int t = 5;
     auto f = [=](){ return t; };
   }
-lambdaExpr(hasAnyCapture(lambdaCapture())) and
-lambdaExpr(hasAnyCapture(lambdaCapture(refersToVarDecl(hasName("t")))))
-  both match `[=](){ return t; }`.
+
+The matcher lambdaExpr(hasAnyCapture(lambdaCapture())) and
+lambdaExpr(hasAnyCapture(lambdaCapture(capturesVar(hasName("t")))))
+  both match [=](){ return t; }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberExpr.html">MemberExpr</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration7')"><a name="hasDeclaration7Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration7"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9169,17 +12184,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9201,11 +12224,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     int m;
     int f(X x) { x.m; return m; }
   };
+
+
+The matcher
 memberExpr(hasObjectExpression(hasType(cxxRecordDecl(hasName("X")))))
-  matches `x.m`, but not `m`; however,
-memberExpr(hasObjectExpression(hasType(pointsTo(
-     cxxRecordDecl(hasName("X"))))))
-  matches `m` (aka. `this-&gt;m`), but not `x.m`.
+matches x.m, but not m; however,
+The matcher memberExpr(hasObjectExpression(hasType(pointsTo(
+cxxRecordDecl(hasName("X"))))))
+matches m (aka. this-&gt;m), but not x.m.
 </pre></td></tr>
 
 
@@ -9214,13 +12240,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 given matcher.
 
 Given
-  struct { int first, second; } first, second;
-  int i(second.first);
-  int j(first.second);
-memberExpr(member(hasName("first")))
-  matches second.first
-  but not first.second (because the member name there is "second").
-</pre></td></tr>
+  struct { int first = 0, second = 1; } first, second;
+  int i = second.first;
+  int j = first.second;
+
+
+The matcher memberExpr(member(hasName("first")))
+matches second.first
+but not </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;</td><td class="name" onclick="toggle('pointee1')"><a name="pointee1Anchor">pointee</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
@@ -9229,10 +12256,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int *a;
-  int const *b;
-  float const *f;
-pointerType(pointee(isConstQualified(), isInteger()))
-  matches "int const *b"
+  const int *b;
+  int * const c = nullptr;
+  const float *f;
+
+The matcher pointerType(pointee(isConstQualified(), isInteger()))
+matches const int *,
+but does not match int * const
+or const float *.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
@@ -9246,9 +12277,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   namespace N { template&lt;class T&gt; void f(T t); }
   template &lt;class T&gt; void g() { using N::f; f(T()); }
-unresolvedLookupExpr(hasAnyDeclaration(
+
+The matcher unresolvedLookupExpr(hasAnyDeclaration(
     namedDecl(hasUnderlyingDecl(hasName("::N::f")))))
-  matches the use of f in g() .
+  matches f in g().
 </pre></td></tr>
 
 
@@ -9258,14 +12290,29 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   struct A { struct B { struct C {}; }; };
   A::B::C c;
-nestedNameSpecifierLoc(hasPrefix(loc(specifiesType(asString("struct A")))))
-  matches "A::"
+
+The matcher
+nestedNameSpecifierLoc(hasPrefix(loc(specifiesType(asString(
+"struct A"))))) matches A::B::.
 </pre></td></tr>
 
 
-<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifierLoc.html">NestedNameSpecifierLoc</a>&gt;</td><td class="name" onclick="toggle('loc1')"><a name="loc1Anchor">loc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifier.html">NestedNameSpecifier</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="loc1"><pre>Matches NestedNameSpecifierLocs for which the given inner
-NestedNameSpecifier-matcher matches.
+<tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifierLoc.html">NestedNameSpecifierLoc</a>&gt;</td><td class="name" onclick="toggle('loc1')"><a name="loc1Anchor">loc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NestedNameSpecifier.html">NestedNameSpecifier</a>&gt; InnerMatcher</td></tr>
+<tr><td colspan="4" class="doc" id="loc1"><pre>Matches NestedNameSpecifierLocs for which the given inner
+NestedNameSpecifier-matcher matches.
+
+Given
+  namespace ns {
+    struct A { static void f(); };
+    void A::f() {}
+    void g() { A::f(); }
+  }
+  ns::A a;
+
+
+The matcher nestedNameSpecifierLoc(loc(specifiesType(
+hasDeclaration(namedDecl(hasName("A")))))) matches A::
+twice.
 </pre></td></tr>
 
 
@@ -9276,9 +12323,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   struct A { struct B { struct C {}; }; };
   A::B::C c;
-nestedNameSpecifierLoc(specifiesTypeLoc(loc(type(
+
+The matcher nestedNameSpecifierLoc(specifiesTypeLoc(loc(qualType(
   hasDeclaration(cxxRecordDecl(hasName("A")))))))
-  matches "A::"
+matches A::
 </pre></td></tr>
 
 
@@ -9288,8 +12336,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   struct A { struct B { struct C {}; }; };
   A::B::C c;
-nestedNameSpecifier(hasPrefix(specifiesType(asString("struct A")))) and
-  matches "A::"
+
+The matcher
+nestedNameSpecifier(hasPrefix(specifiesType(asString(
+"struct A")))) matches struct A::B
 </pre></td></tr>
 
 
@@ -9300,8 +12350,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   namespace ns { struct A {}; }
   ns::A a;
-nestedNameSpecifier(specifiesNamespace(hasName("ns")))
-  matches "ns::"
+
+The matcher
+nestedNameSpecifier(specifiesNamespace(hasName("ns"))) matches
+ns.
 </pre></td></tr>
 
 
@@ -9312,10 +12364,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   struct A { struct B { struct C {}; }; };
   A::B::C c;
-nestedNameSpecifier(specifiesType(
+
+The matcher nestedNameSpecifier(specifiesType(
   hasDeclaration(cxxRecordDecl(hasName("A")))
 ))
-  matches "A::"
+matches A.
 </pre></td></tr>
 
 
@@ -9323,12 +12376,16 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasAnyClause0"><pre>Matches any clause in an OpenMP directive.
 
 Given
-
+  void foo() {
   #pragma omp parallel
+    ;
   #pragma omp parallel default(none)
+    ;
+  }
 
-``ompExecutableDirective(hasAnyClause(anything()))`` matches
-``omp parallel default(none)``.
+
+The matcher ompExecutableDirective(hasAnyClause(anything()))
+matches #pragma omp parallel default(none).
 </pre></td></tr>
 
 
@@ -9339,13 +12396,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 If it is, it will never match.
 
 Given
+  void foo() {
+    #pragma omp parallel
+    ;
+    #pragma omp parallel
+    {}
+  }
 
-   #pragma omp parallel
-   ;
-   #pragma omp parallel
-   {}
 
-``ompExecutableDirective(hasStructuredBlock(nullStmt()))`` will match ``;``
+The matcher
+ompExecutableDirective(hasStructuredBlock(nullStmt().bind("stmt")))
+matches #pragma omp parallel,
+with stmtt() matching {}.
 </pre></td></tr>
 
 
@@ -9357,22 +12419,29 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Note that a class is not considered to be derived from itself.
 
 Example matches Y, Z, C (Base == hasName("X"))
-  class X;
+  class X {};
   class Y : public X {};  // directly derived
   class Z : public Y {};  // indirectly derived
   typedef X A;
   typedef A B;
   class C : public B {};  // derived from a typedef of X
 
-In the following example, Bar matches isDerivedFrom(hasName("X")):
-  class Foo;
-  typedef Foo X;
-  class Bar : public Foo {};  // derived from a type that X is a typedef of
+  class Foo {};
+  typedef Foo Alias;
+  class Bar : public Alias {};
+  // derived from a type that Alias is a typedef of Foo
+
+
+The matcher cxxRecordDecl(isDerivedFrom(hasName("X")))
+matches Y, Z and C.
+The matcher cxxRecordDecl(isDerivedFrom(hasName("Foo")))
+matches Bar.
 
 In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
   @interface NSObject @end
   @interface Bar : NSObject @end
 
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXRecordDecl.html">CXXRecordDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;
 </pre></td></tr>
 
@@ -9383,24 +12452,45 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Note that a class is not considered to be derived from itself.
 
-Example matches Y, C (Base == hasName("X"))
-  class X;
+Given
+  class X {};
   class Y : public X {};  // directly derived
   class Z : public Y {};  // indirectly derived
   typedef X A;
   typedef A B;
   class C : public B {};  // derived from a typedef of X
 
+The matcher
+cxxRecordDecl(isDirectlyDerivedFrom(namedDecl(hasName("X"))))
+matches Y and C (Base == hasName("X")
+
 In the following example, Bar matches isDerivedFrom(hasName("X")):
-  class Foo;
+  class Foo {};
   typedef Foo X;
   class Bar : public Foo {};  // derived from a type that X is a typedef of
+
+The matcher cxxRecordDecl(isDerivedFrom(hasName("X")))
+matches Bar
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCInterfaceDecl.html">ObjCInterfaceDecl</a>&gt;</td><td class="name" onclick="toggle('isSameOrDerivedFrom1')"><a name="isSameOrDerivedFrom1Anchor">isSameOrDerivedFrom</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1NamedDecl.html">NamedDecl</a>&gt; Base</td></tr>
 <tr><td colspan="4" class="doc" id="isSameOrDerivedFrom1"><pre>Similar to isDerivedFrom(), but also matches classes that directly
 match Base.
+
+Given
+  class X {};
+  class Y : public X {};  // directly derived
+  class Z : public Y {};  // indirectly derived
+  typedef X A;
+  typedef A B;
+  class C : public B {};  // derived from a typedef of X
+
+The matcher
+cxxRecordDecl(isSameOrDerivedFrom(cxxRecordDecl(hasName("X"))),
+isDefinition())
+matches class X {}, class Y : public X {},
+class Z : public Y {} and class C : public B {}.
 </pre></td></tr>
 
 
@@ -9409,19 +12499,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 given matcher; or 2) if the Obj-C message expression's callee's method
 declaration matches the given matcher.
 
-Example matches y.x() (matcher = callExpr(callee(
-                                   cxxMethodDecl(hasName("x")))))
+Example 1
   class Y { public: void x(); };
   void z() { Y y; y.x(); }
 
-Example 2. Matches [I foo] with
-objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))
+The matcher callExpr(callee(cxxMethodDecl(hasName("x"))))
+matches y.x()
 
+Example 2
   @interface I: NSObject
   +(void)foo;
   @end
   ...
   [I foo]
+
+The matcher
+objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))
+matches [I foo]
 </pre></td></tr>
 
 
@@ -9430,17 +12524,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 expression, or an ObjC-message-send expression.
 
 Given
-  void x(int, int, int) { int y; x(1, y, 42); }
-callExpr(hasAnyArgument(declRefExpr()))
-  matches x(1, y, 42)
-with hasAnyArgument(...)
+  void x(int, int, int) { int y = 42; x(1, y, 42); }
+The matcher
+callExpr(hasAnyArgument(ignoringImplicit(declRefExpr()))) matches
+x(1, y, 42) with hasAnyArgument(...)
   matching y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
   void foo(I *i) { [i f:12]; }
+
+The matcher
 objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))
-  matches [i f:12]
+matches [i f:12]
 </pre></td></tr>
 
 
@@ -9448,9 +12544,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasArgument3"><pre>Matches the n'th argument of a call expression or a constructor
 call expression.
 
-Example matches y in x(y)
-    (matcher = callExpr(hasArgument(0, declRefExpr())))
+Given
   void x(int) { int y; x(y); }
+The matcher callExpr(hasArgument(0, declRefExpr().bind("arg")))
+matches x(y),
+with declRefExpr() matching y.
 </pre></td></tr>
 
 
@@ -9458,23 +12556,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasReceiver0"><pre>Matches if the Objective-C message is sent to an instance,
 and the inner matcher matches on that instance.
 
-For example the method call in
+Given
   NSString *x = @"hello";
   [x containsString:@"h"];
-is matched by
+
+The matcher
 objcMessageExpr(hasReceiver(declRefExpr(to(varDecl(hasName("x"))))))
+matches [x containsString:@"h"];
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCMessageExpr.html">ObjCMessageExpr</a>&gt;</td><td class="name" onclick="toggle('hasReceiverType0')"><a name="hasReceiverType0Anchor">hasReceiverType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasReceiverType0"><pre>Matches on the receiver of an ObjectiveC Message expression.
 
-Example
-matcher = objCMessageExpr(hasReceiverType(asString("UIWebView *")));
-matches the [webView ...] message invocation.
   NSString *webViewJavaScript = ...
   UIWebView *webView = ...
   [webView stringByEvaluatingJavaScriptFromString:webViewJavascript];
+
+The matcher objCMessageExpr(hasReceiverType(asString("UIWebView
+*"))) matches
+[webViewstringByEvaluatingJavaScriptFromString:webViewJavascript];
 </pre></td></tr>
 
 
@@ -9486,23 +12587,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x, int y, int z) {} };
-cxxMethodDecl(hasAnyParameter(hasName("y")))
-  matches f(int x, int y, int z) {}
+
+The matcher cxxMethodDecl(hasAnyParameter(hasName("y")))
+  matches f
 with hasAnyParameter(...)
   matching int y
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
+
 the matcher objcMethodDecl(hasAnyParameter(hasName("y")))
-matches the declaration of method f with hasParameter
+  matches the declaration of method f with hasParameter
 matching y.
 
 For blocks, given
   b = ^(int y) { printf("%d", y) };
 
+
 the matcher blockDecl(hasAnyParameter(hasName("y")))
-matches the declaration of the block b with hasParameter
+  matches the declaration of the block b with hasParameter
 matching y.
 </pre></td></tr>
 
@@ -9513,15 +12617,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   class X { void f(int x) {} };
-cxxMethodDecl(hasParameter(0, hasType(varDecl())))
-  matches f(int x) {}
+
+The matcher
+cxxMethodDecl(hasParameter(0, hasType(asString("int"))))
+matches f
 with hasParameter(...)
-  matching int x
+matching int x.
 
 For ObjectiveC, given
   @interface I - (void) f:(int) y; @end
 
-the matcher objcMethodDecl(hasParameter(0, hasName("y")))
+
+The matcher objcMethodDecl(hasParameter(0, hasName("y")))
 matches the declaration of method f with hasParameter
 matching y.
 </pre></td></tr>
@@ -9530,19 +12637,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ObjCPropertyDecl.html">ObjCPropertyDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc10')"><a name="hasTypeLoc10Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc10"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -9559,14 +12673,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasSourceExpression1"><pre>Matches if the cast's source expression
 or opaque value's source expression matches the given matcher.
 
-Example 1: matches "a string"
-(matcher = castExpr(hasSourceExpression(cxxConstructExpr())))
-class URL { URL(string); };
-URL url = "a string";
+Given
+ struct URL { URL(const char*); };
+ URL url = "a string";
+
+The matcher castExpr(hasSourceExpression(cxxConstructExpr()))
+matches "a string".
 
-Example 2: matches 'b' (matcher =
-opaqueValueExpr(hasSourceExpression(implicitCastExpr(declRefExpr())))
-int a = b ?: 1;
+Given
+void foo(bool b) {
+  int a = b ?: 1;
+}
+
+The matcher
+opaqueValueExpr(hasSourceExpression(
+              implicitCastExpr(has(
+                implicitCastExpr(has(declRefExpr()))))))
+matches b twice, for the conditiona and the true expression.
 </pre></td></tr>
 
 
@@ -9581,9 +12704,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     foo(t);
     bar(t);
   }
-unresolvedLookupExpr(hasAnyDeclaration(
+
+The matcher unresolvedLookupExpr(hasAnyDeclaration(
     functionTemplateDecl(hasName("foo"))))
-  matches foo in foo(t); but not bar in bar(t);
+matches foo in foo(t);
+but does not match bar in bar(t);
 </pre></td></tr>
 
 
@@ -9594,8 +12719,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   int (*ptr_to_array)[4];
   int (*ptr_to_func)(int);
 
-varDecl(hasType(pointsTo(parenType(innerType(functionType()))))) matches
-ptr_to_func but not ptr_to_array.
+The matcher
+varDecl(hasType(pointsTo(parenType(innerType(functionType())))))
+  matches ptr_to_func but not
+  ptr_to_array.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ParenType.html">ParenType</a>&gt;
 </pre></td></tr>
@@ -9607,8 +12734,8 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int* x;
-pointerTypeLoc(hasPointeeLoc(loc(asString("int"))))
-  matches `int*`.
+The matcher pointerTypeLoc(hasPointeeLoc(loc(asString("int"))))
+  matches int*.
 </pre></td></tr>
 
 
@@ -9618,10 +12745,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int *a;
-  int const *b;
-  float const *f;
-pointerType(pointee(isConstQualified(), isInteger()))
-  matches "int const *b"
+  const int *b;
+  int * const c = nullptr;
+  const float *f;
+
+The matcher pointerType(pointee(isConstQualified(), isInteger()))
+matches const int *,
+but does not match int * const
+or const float *.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
@@ -9631,19 +12762,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('hasCanonicalType0')"><a name="hasCanonicalType0Anchor">hasCanonicalType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasCanonicalType0"><pre>Matches QualTypes whose canonical type matches InnerMatcher.
 
-Given:
+Given
   typedef int &amp;int_ref;
   int a;
   int_ref b = a;
 
-varDecl(hasType(qualType(referenceType()))))) will not match the
-declaration of b but varDecl(hasType(qualType(hasCanonicalType(referenceType())))))) does.
+The matcher varDecl(hasType(qualType(referenceType())))
+does not match int_ref b = a,
+but the matcher
+varDecl(hasType(qualType(hasCanonicalType(referenceType()))))
+does match int_ref b = a.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration6')"><a name="hasDeclaration6Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration6"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9653,17 +12787,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9681,30 +12823,56 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   void (*fp)(void);
 The matcher
-  varDecl(hasType(pointerType(pointee(ignoringParens(functionType())))))
-would match the declaration for fp.
+varDecl(hasType(pointerType(pointee(ignoringParens(functionType())))))
+matches fp.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('pointsTo1')"><a name="pointsTo1Anchor">pointsTo</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="pointsTo1"><pre>Overloaded to match the pointee type's declaration.
+<tr><td colspan="4" class="doc" id="pointsTo1"><pre>Matches if the matched type is a pointer type and the pointee type
+  matches the specified matcher.
+Overloaded to match the pointee type's declaration.
+
+Given
+  class Y { public: void x(); };
+  void z() { Y *y; y-&gt;x(); }
+
+The matcher cxxMemberCallExpr(on(hasType(pointsTo(
+     cxxRecordDecl(hasName("Y"))))))
+matches y-&gt;x()
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('pointsTo0')"><a name="pointsTo0Anchor">pointsTo</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="pointsTo0"><pre>Matches if the matched type is a pointer type and the pointee type
-matches the specified matcher.
+  matches the specified matcher.
 
-Example matches y-&gt;x()
-  (matcher = cxxMemberCallExpr(on(hasType(pointsTo
-     cxxRecordDecl(hasName("Y")))))))
+Given
   class Y { public: void x(); };
   void z() { Y *y; y-&gt;x(); }
+
+The matcher cxxMemberCallExpr(on(hasType(pointsTo(
+     qualType()))))
+matches y-&gt;x()
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt;</td><td class="name" onclick="toggle('references1')"><a name="references1Anchor">references</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt; InnerMatcher</td></tr>
-<tr><td colspan="4" class="doc" id="references1"><pre>Overloaded to match the referenced type's declaration.
+<tr><td colspan="4" class="doc" id="references1"><pre>Matches if the matched type is a reference type and the referenced
+type matches the specified matcher.
+Overloaded to match the referenced type's declaration.
+
+Given
+  class X {
+    void a(X b) {
+      X &amp;x = b;
+      const X &amp;y = b;
+    }
+  };
+
+The matcher
+varDecl(hasType(references(cxxRecordDecl(hasName("X"))))) matches
+X &amp;x = b and const X &amp;y = b.
 </pre></td></tr>
 
 
@@ -9712,14 +12880,17 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="references0"><pre>Matches if the matched type is a reference type and the referenced
 type matches the specified matcher.
 
-Example matches X &amp;x and const X &amp;y
-    (matcher = varDecl(hasType(references(cxxRecordDecl(hasName("X"))))))
+Given
   class X {
     void a(X b) {
       X &amp;x = b;
       const X &amp;y = b;
     }
   };
+
+The matcher
+varDecl(hasType(references(qualType()))) matches
+X &amp;x = b and const X &amp;y = b.
 </pre></td></tr>
 
 
@@ -9728,16 +12899,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 `InnerMatcher`.
 
 Given
-  int* const x;
-  const int y;
-qualifiedTypeLoc(hasUnqualifiedLoc(pointerTypeLoc()))
-  matches the `TypeLoc` of the variable declaration of `x`, but not `y`.
-</pre></td></tr>
+  int* const x = nullptr;
+  const int y = 0;
+
+
+The matcher qualifiedTypeLoc(hasUnqualifiedLoc(pointerTypeLoc()))
+matches the type int* of the variable declaration but
+not </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1RecordType.html">RecordType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration5')"><a name="hasDeclaration5Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration5"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9747,17 +12920,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9776,8 +12957,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   int x = 3;
   int&amp; xx = x;
-referenceTypeLoc(hasReferentLoc(loc(asString("int"))))
-  matches `int&amp;`.
+
+
+The matcher referenceTypeLoc(hasReferentLoc(loc(asString("int"))))
+  matches int&amp;.
 </pre></td></tr>
 
 
@@ -9787,10 +12970,14 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int *a;
-  int const *b;
-  float const *f;
-pointerType(pointee(isConstQualified(), isInteger()))
-  matches "int const *b"
+  const int *b;
+  int * const c = nullptr;
+  const float *f;
+
+The matcher pointerType(pointee(isConstQualified(), isInteger()))
+matches const int *,
+but does not match int * const
+or const float *.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockPointerType.html">BlockPointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1MemberPointerType.html">MemberPointerType</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1PointerType.html">PointerType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ReferenceType.html">ReferenceType</a>&gt;
@@ -9801,11 +12988,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasReturnValue0"><pre>Matches the return value expression of a return statement
 
 Given
-  return a + b;
-hasReturnValue(binaryOperator())
-  matches 'return a + b'
-with binaryOperator()
-  matching 'a + b'
+  int foo(int a, int b) {
+    return a + b;
+  }
+The matcher
+returnStmt(hasReturnValue(binaryOperator().bind("op"))) matches
+return a + b, with binaryOperator() matching
+a + b.
 </pre></td></tr>
 
 
@@ -9814,17 +13003,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 a given matcher. Also matches StmtExprs that have CompoundStmt as children.
 
 Given
-  { {}; 1+2; }
-hasAnySubstatement(compoundStmt())
-  matches '{ {}; 1+2; }'
+void foo() { { {}; 1+2; } }
+The matcher
+compoundStmt(hasAnySubstatement(compoundStmt().bind("compound")))
+{ {}; 1+2; } and { { {}; 1+2; } }
 with compoundStmt()
-  matching '{}'
+matching {} and { {}; 1+2; }.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('alignOfExpr0')"><a name="alignOfExpr0Anchor">alignOfExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnaryExprOrTypeTraitExpr.html">UnaryExprOrTypeTraitExpr</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="alignOfExpr0"><pre>Same as unaryExprOrTypeTraitExpr, but only matching
 alignof.
+
+Given
+  int align = alignof(int);
+
+
+The matcher alignOfExpr(expr())
+matches alignof(int).
 </pre></td></tr>
 
 
@@ -9832,26 +13029,30 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="forCallable0"><pre>Matches declaration of the function, method, or block the statement
 belongs to.
 
-Given:
-F&amp; operator=(const F&amp; o) {
-  std::copy_if(o.begin(), o.end(), begin(), [](V v) { return v &gt; 0; });
-  return *this;
-}
-returnStmt(forCallable(functionDecl(hasName("operator="))))
-  matches 'return *this'
-  but does not match 'return v &gt; 0'
+Given
+struct F {
+  F&amp; operator=(const F&amp; other) {
+    []() { return 42 == 42; };
+    return *this;
+  }
+};
 
-Given:
--(void) foo {
+The matcher returnStmt(forFunction(hasName("operator=")))
+matches return *this
+but does not match return 42 == 42.
+
+Given
+void foo {
   int x = 1;
   dispatch_sync(queue, ^{ int y = 2; });
 }
-declStmt(forCallable(objcMethodDecl()))
-  matches 'int x = 1'
-  but does not match 'int y = 2'.
-whereas declStmt(forCallable(blockDecl()))
-  matches 'int y = 2'
-  but does not match 'int x = 1'.
+
+The matcher declStmt(forCallable(objcMethodDecl()))
+matches int x = 1
+but does not match int y = 2.
+The matcher declStmt(forCallable(blockDecl()))
+matches int y = 2
+but does not match int x = 1.
 </pre></td></tr>
 
 
@@ -9860,23 +13061,34 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Deprecated. Use forCallable() to correctly handle the situation when
 the declaration is not a function (but a block or an Objective-C method).
-forFunction() not only fails to take non-functions into account but also
-may match the wrong declaration in their presence.
+The matcher forFunction() not only fails to take non-functions
+into account but also may match the wrong declaration in their presence.
 
-Given:
-F&amp; operator=(const F&amp; o) {
-  std::copy_if(o.begin(), o.end(), begin(), [](V v) { return v &gt; 0; });
-  return *this;
-}
-returnStmt(forFunction(hasName("operator=")))
-  matches 'return *this'
-  but does not match 'return v &gt; 0'
+Given
+  struct F {
+    F&amp; operator=(const F&amp; other) {
+      []() { return 42 == 42; };
+      return *this;
+    }
+  };
+
+
+The matcher returnStmt(forFunction(hasName("operator=")))
+matches return *this
+but does not match return 42 == 42.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt;</td><td class="name" onclick="toggle('sizeOfExpr0')"><a name="sizeOfExpr0Anchor">sizeOfExpr</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnaryExprOrTypeTraitExpr.html">UnaryExprOrTypeTraitExpr</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="sizeOfExpr0"><pre>Same as unaryExprOrTypeTraitExpr, but only matching
 sizeof.
+
+Given
+  struct S { double x; double y; };
+  int size = sizeof(struct S);
+
+The matcher sizeOfExpr(expr())
+matches sizeof(struct S).
 </pre></td></tr>
 
 
@@ -9890,7 +13102,9 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   int i;
   double j = F(i);
 
-substTemplateTypeParmType(hasReplacementType(type())) matches int
+
+The matcher substTemplateTypeParmType(hasReplacementType(type()))
+matches int.
 </pre></td></tr>
 
 
@@ -9899,11 +13113,18 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 statement. This matcher may produce multiple matches.
 
 Given
-  switch (1) { case 1: case 2: default: switch (2) { case 3: case 4: ; } }
-switchStmt(forEachSwitchCase(caseStmt().bind("c"))).bind("s")
-  matches four times, with "c" binding each of "case 1:", "case 2:",
-"case 3:" and "case 4:", and "s" respectively binding "switch (1)",
-"switch (1)", "switch (2)" and "switch (2)".
+  void foo() {
+    switch (1) { case 1: case 2: default: switch (2) { case 3: case 4: ; } }
+  }
+The matcher
+switchStmt(forEachSwitchCase(caseStmt().bind("c")))
+matches four times, matching
+switch (1) { case 1: case 2: default: switch (2) { case 3:
+case 4: ; } } and
+switch (2) { case 3: case 4: ; }, with
+caseStmt() matching each of case 1:,
+case 2:, case 3:
+and case 4:.
 </pre></td></tr>
 
 
@@ -9911,15 +13132,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition4"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1SwitchStmt.html">SwitchStmt</a>&gt;</td><td class="name" onclick="toggle('hasInitStatement1')"><a name="hasInitStatement1Anchor">hasInitStatement</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Stmt.html">Stmt</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasInitStatement1"><pre>Matches selection statements with initializer.
 
-Given:
+Given
+ struct vec { int* begin(); int* end(); };
+ int foobar();
+ vec&amp; get_range();
  void foo() {
    if (int i = foobar(); i &gt; 0) {}
    switch (int i = foobar(); i) {}
@@ -9930,18 +13159,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
    switch (foobar()) {}
    for (auto&amp; x : get_range()) {}
  }
-ifStmt(hasInitStatement(anything()))
-  matches the if statement in foo but not in bar.
-switchStmt(hasInitStatement(anything()))
-  matches the switch statement in foo but not in bar.
-cxxForRangeStmt(hasInitStatement(anything()))
-  matches the range for statement in foo but not in bar.
+
+The matcher ifStmt(hasInitStatement(anything()))
+  matches the if statement if (int i = foobar(); i &gt; 0) {}
+  in foo but not if (foobar() &gt; 0) {} in bar.
+The matcher switchStmt(hasInitStatement(anything()))
+  matches the switch statement switch (int i = foobar(); i) {}
+  in foo but not switch (foobar()) {} in bar.
+The matcher cxxForRangeStmt(hasInitStatement(anything()))
+  matches the range for statement
+  for (auto&amp; a = get_range(); auto&amp; x : a) {} in foo
+  but not for (auto&amp; x : get_range()) {} in bar.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TagType.html">TagType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration4')"><a name="hasDeclaration4Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration4"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -9951,17 +13185,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -9976,19 +13218,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateArgumentLoc.html">TemplateArgumentLoc</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc11')"><a name="hasTypeLoc11Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc11"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -10008,10 +13257,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   struct B { int next; };
   template&lt;int(B::*next_ptr)&gt; struct A {};
   A&lt;&amp;B::next&gt; a;
+
+The matcher
 templateSpecializationType(hasAnyTemplateArgument(
-  isExpr(hasDescendant(declRefExpr(to(fieldDecl(hasName("next"))))))))
-  matches the specialization A&lt;&amp;B::next&gt; with fieldDecl(...) matching
-    B::next
+  isExpr(hasDescendant(declRefExpr(to(fieldDecl(hasName("next")).bind("next")))))))
+matches the specialization A&lt;&amp;struct B::next&gt;
+with fieldDecl(hasName("next")) matching
+B::next.
 </pre></td></tr>
 
 
@@ -10023,10 +13275,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   struct B { int next; };
   template&lt;int(B::*next_ptr)&gt; struct A {};
   A&lt;&amp;B::next&gt; a;
+
+The matcher
 classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToDeclaration(fieldDecl(hasName("next")))))
-  matches the specialization A&lt;&amp;B::next&gt; with fieldDecl(...) matching
-    B::next
+    refersToDeclaration(fieldDecl(hasName("next")).bind("next"))))
+matches the specialization struct A&lt;&amp;B::next&gt;
+with fieldDecl(hasName("next")) matching
+B::next.
 </pre></td></tr>
 
 
@@ -10036,9 +13291,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;int T&gt; struct C {};
   C&lt;42&gt; c;
-classTemplateSpecializationDecl(
+
+The matcher classTemplateSpecializationDecl(
   hasAnyTemplateArgument(refersToIntegralType(asString("int"))))
-  matches the implicit instantiation of C in C&lt;42&gt;.
+matches the implicitly declared specialization
+struct C&lt;42&gt; from the instantiation for the type of the
+variable c .
 </pre></td></tr>
 
 
@@ -10049,9 +13307,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;template &lt;typename&gt; class S&gt; class X {};
   template&lt;typename T&gt; class Y {};
   X&lt;Y&gt; xi;
+
+The matcher
 classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToTemplate(templateName())))
-  matches the specialization X&lt;Y&gt;
+              refersToTemplate(templateName())))
+matches the specialization class X&lt;Y&gt;
 </pre></td></tr>
 
 
@@ -10062,9 +13322,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   struct X {};
   template&lt;typename T&gt; struct A {};
   A&lt;X&gt; a;
+
+The matcher
 classTemplateSpecializationDecl(hasAnyTemplateArgument(refersToType(
-  recordType(hasDeclaration(recordDecl(hasName("X")))))))
-matches the specialization of struct A generated by A&lt;X&gt;.
+            recordType(hasDeclaration(recordDecl(hasName("X")))))))
+matches the specialization struct A&lt;struct X&gt;.
 </pre></td></tr>
 
 
@@ -10077,9 +13339,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;typename T&gt; class A {};
   A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+hasTypeLoc(loc(asString("int"))))))))) matches A&lt;int&gt; a.
 </pre></td></tr>
 
 
@@ -10092,9 +13356,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T, typename U&gt; class A {};
   A&lt;double, int&gt; b;
   A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+hasTypeLoc(loc(asString("double")))))))))
+matches A&lt;double, int&gt; b, but not double&gt; c}.
 </pre></td></tr>
 
 
@@ -10114,13 +13381,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template &lt;typename T, typename U&gt;
   void f(T&amp;&amp; t, U&amp;&amp; u) {}
 
-  bool B = false;
-  f(R, B);
-templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-  matches twice, with expr() matching 'R * 2' and 'R * 4'
-functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
-  and 'bool'
+  void foo() {
+    bool B = false;
+    f(R, B);
+  }
+
+The matcher
+templateSpecializationType(forEachTemplateArgument(isExpr(expr().bind("t_arg"))))
+matches Matrix&lt;int, R * 2, R * 4&gt; twice, with
+expr() matching R * 2 and
+R * 4.
+The matcher
+functionDecl(forEachTemplateArgument(refersToType(qualType().bind("type"))))
+matches the specialization of f twice,
+with qualType() matching
+unsigned and
+bool.
 </pre></td></tr>
 
 
@@ -10134,21 +13410,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;&gt; class A&lt;double&gt; {};
   A&lt;int&gt; a;
 
-  template&lt;typename T&gt; f() {};
+  template&lt;typename T&gt; void f() {};
   void func() { f&lt;int&gt;(); };
 
-classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToType(asString("int"))))
-  matches the specialization A&lt;int&gt;
 
-functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher classTemplateSpecializationDecl(
+                        hasAnyTemplateArgument(
+                          refersToType(asString("int"))))
+matches class A&lt;int&gt;.
+
+The matcher
+functionDecl(hasAnyTemplateArgument(
+              refersToType(asString("int"))))
+matches the instantiation of f.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateSpecializationType.html">TemplateSpecializationType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration3')"><a name="hasDeclaration3Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration3"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -10158,17 +13438,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -10187,23 +13475,26 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   template&lt;typename T, typename U&gt; class A {};
-  A&lt;bool, int&gt; b;
-  A&lt;int, bool&gt; c;
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
 
   template&lt;typename T&gt; void f() {}
   void func() { f&lt;int&gt;(); };
+
+The matcher
 classTemplateSpecializationDecl(hasTemplateArgument(
     1, refersToType(asString("int"))))
-  matches the specialization A&lt;bool, int&gt;
+matches the specialization class A&lt;double, int&gt;.
 
-functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher functionDecl(hasTemplateArgument(0,
+                        refersToType(asString("int"))))
+matches the specialization of f.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TemplateTypeParmType.html">TemplateTypeParmType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration2')"><a name="hasDeclaration2Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration2"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -10213,17 +13504,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -10238,25 +13537,37 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt;</td><td class="name" onclick="toggle('loc0')"><a name="loc0Anchor">loc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1QualType.html">QualType</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="loc0"><pre>Matches TypeLocs for which the given inner
 QualType-matcher matches.
+
+  int a = 10;
+
+The matcher typeLoc(loc(qualType(isInteger())))
+matches the int of a .
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefNameDecl.html">TypedefNameDecl</a>&gt;</td><td class="name" onclick="toggle('hasTypeLoc12')"><a name="hasTypeLoc12Anchor">hasTypeLoc</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypeLoc.html">TypeLoc</a>&gt; Inner</td></tr>
 <tr><td colspan="4" class="doc" id="hasTypeLoc12"><pre>Matches if the type location of a node matches the inner matcher.
 
-Examples:
+Given
   int x;
-declaratorDecl(hasTypeLoc(loc(asString("int"))))
-  matches int x
+The matcher declaratorDecl(hasTypeLoc(loc(asString("int"))))
+matches int x.
+
+Given
+struct point { point(double, double); };
+point p = point(1.0, -1.0);
 
-auto x = int(3);
-cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-  matches int(3)
+The matcher
+cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))
+matches point(1.0, -1.0).
 
+Given
 struct Foo { Foo(int, int); };
-auto x = Foo(1, 2);
-cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-  matches Foo(1, 2)
+Foo x = Foo(1, 2);
+
+The matcher cxxTemporaryObjectExpr(hasTypeLoc(
+                          loc(asString("Foo"))))
+matches Foo(1, 2).
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1BlockDecl.html">BlockDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXCtorInitializer.html">CXXCtorInitializer</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXFunctionalCastExpr.html">CXXFunctionalCastExpr</a>&gt;,
@@ -10273,23 +13584,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasType2"><pre>Matches if the expression's or declaration's type matches a type
 matcher.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and U (matcher = typedefDecl(hasType(asString("int")))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              asString("class X")))
+Exmaple
  class X {};
  void y(X &amp;x) { x; X z; }
  typedef int U;
  class Y { friend class X; };
  class Z : public virtual X {};
+
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z
+The matcher typedefDecl(hasType(asString("int")))
+matches typedef int U
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1TypedefType.html">TypedefType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration1')"><a name="hasDeclaration1Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration1"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -10299,17 +13618,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -10328,8 +13655,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 For example, in:
   class A {};
   using B = A;
-The matcher type(hasUnqualifiedDesugaredType(recordType())) matches
-both B and A.
+  B b;
+
+The matcher
+varDecl(hasType(hasUnqualifiedDesugaredType(recordType())))
+matches B b.
 </pre></td></tr>
 
 
@@ -10338,17 +13668,23 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   int a, c; float b; int s = sizeof(a) + sizeof(b) + alignof(c);
-unaryExprOrTypeTraitExpr(hasArgumentOfType(asString("int"))
-  matches sizeof(a) and alignof(c)
+
+The matcher
+unaryExprOrTypeTraitExpr(hasArgumentOfType(asString("int")))
+matches sizeof(a) and alignof(c)
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnaryOperator.html">UnaryOperator</a>&gt;</td><td class="name" onclick="toggle('hasUnaryOperand0')"><a name="hasUnaryOperand0Anchor">hasUnaryOperand</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt; InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasUnaryOperand0"><pre>Matches if the operand of a unary operator matches.
 
-Example matches true (matcher = hasUnaryOperand(
-                                  cxxBoolLiteral(equals(true))))
-  !true
+void foo() {
+  !true;
+}
+
+The matcher
+unaryOperator(hasUnaryOperand(cxxBoolLiteral(equals(true))))
+matches !true.
 </pre></td></tr>
 
 
@@ -10362,17 +13698,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
     int m;
     int f(X x) { x.m; return m; }
   };
+
+
+The matcher
 memberExpr(hasObjectExpression(hasType(cxxRecordDecl(hasName("X")))))
-  matches `x.m`, but not `m`; however,
-memberExpr(hasObjectExpression(hasType(pointsTo(
-     cxxRecordDecl(hasName("X"))))))
-  matches `m` (aka. `this-&gt;m`), but not `x.m`.
+matches x.m, but not m; however,
+The matcher memberExpr(hasObjectExpression(hasType(pointsTo(
+cxxRecordDecl(hasName("X"))))))
+matches m (aka. this-&gt;m), but not x.m.
 </pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UnresolvedUsingType.html">UnresolvedUsingType</a>&gt;</td><td class="name" onclick="toggle('hasDeclaration0')"><a name="hasDeclaration0Anchor">hasDeclaration</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Decl.html">Decl</a>&gt;  InnerMatcher</td></tr>
 <tr><td colspan="4" class="doc" id="hasDeclaration0"><pre>Matches a node if the declaration associated with that node
-matches the given matcher.
+  matches the given matcher.
 
 The associated declaration is:
 - for type nodes, the declaration of the underlying type
@@ -10382,17 +13721,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 - for CXXNewExpr, the declaration of the operator new
 - for ObjCIvarExpr, the declaration of the ivar
 
-For type nodes, hasDeclaration will generally match the declaration of the
-sugared type. Given
+Given
   class X {};
   typedef X Y;
   Y y;
-in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-typedefDecl. A common use case is to match the underlying, desugared type.
+
+For type nodes, hasDeclaration will generally match the declaration of the
+sugared type, i.e., the matcher
+varDecl(hasType(qualType(hasDeclaration(decl().bind("d"))))),
+matches Y y, with
+the matcher decl() matching
+typedef X Y;.
+A common use case is to match the underlying, desugared type.
 This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-  varDecl(hasType(hasUnqualifiedDesugaredType(
-      recordType(hasDeclaration(decl())))))
-In this matcher, the decl will match the CXXRecordDecl of class X.
+varDecl(hasType(hasUnqualifiedDesugaredType(
+      recordType(hasDeclaration(decl().bind("d"))))))
+matches Y y.
+In this matcher, the matcher decl() will match the
+CXXRecordDecl
+class X {};.
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1AddrLabelExpr.html">AddrLabelExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CallExpr.html">CallExpr</a>&gt;,
   Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXConstructExpr.html">CXXConstructExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXNewExpr.html">CXXNewExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;,
@@ -10412,8 +13759,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   namespace X { int a; void b(); }
   using X::a;
   using X::b;
+
+The matcher
 usingDecl(hasAnyUsingShadowDecl(hasTargetDecl(functionDecl())))
-  matches using X::b but not using X::a </pre></td></tr>
+  matches using X::b
+  but not X::a}
+</pre></td></tr>
 
 
 <tr><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingType.html">UsingType</a>&gt;</td><td class="name" onclick="toggle('hasUnderlyingType1')"><a name="hasUnderlyingType1Anchor">hasUnderlyingType</a></td><td>Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Type.html">Type</a>&gt;</td></tr>
@@ -10422,8 +13773,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   decltype(1) a = 1;
   decltype(2.0) b = 2.0;
-decltypeType(hasUnderlyingType(isInteger()))
-  matches the type of "a"
+
+
+The matcher decltypeType(hasUnderlyingType(isInteger()))
+matches the type decltype(1) of the variable
+declaration of a .
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DecltypeType.html">DecltypeType</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingType.html">UsingType</a>&gt;
 </pre></td></tr>
@@ -10433,18 +13787,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="throughUsingDecl1"><pre>Matches if a node refers to a declaration through a specific
 using shadow declaration.
 
-Examples:
+Given
   namespace a { int f(); }
   using a::f;
   int x = f();
-declRefExpr(throughUsingDecl(anything()))
-  matches f
+
+The matcher declRefExpr(throughUsingDecl(anything()))
+matches f
 
   namespace a { class X{}; }
   using a::X;
   X x;
-typeLoc(loc(usingType(throughUsingDecl(anything()))))
-  matches X
+
+The matcher typeLoc(loc(usingType(throughUsingDecl(anything()))))
+matches X
 
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1DeclRefExpr.html">DeclRefExpr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1UsingType.html">UsingType</a>&gt;
 </pre></td></tr>
@@ -10460,21 +13816,31 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
 declaration of x.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              cxxRecordDecl(hasName("X"))))
  class X {};
  void y(X &amp;x) { x; X z; }
  class Y { friend class X; };
  class Z : public virtual X {};
 
-Example matches class Derived
-(matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z.
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X.
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches
+class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
+
+Given
 class Base {};
 class Derived : Base {};
 
+The matcher
+cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base")))))
+matches class Derived : Base {}.
+
 Usable as: Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1Expr.html">Expr</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1FriendDecl.html">FriendDecl</a>&gt;, Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1ValueDecl.html">ValueDecl</a>&gt;,
 Matcher&lt;<a href="https://clang.llvm.org/doxygen/classclang_1_1CXXBaseSpecifier.html">CXXBaseSpecifier</a>&gt;
 </pre></td></tr>
@@ -10484,17 +13850,25 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasType3"><pre>Matches if the expression's or declaration's type matches a type
 matcher.
 
-Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-            and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-            and U (matcher = typedefDecl(hasType(asString("int")))
-            and friend class X (matcher = friendDecl(hasType("X"))
-            and public virtual X (matcher = cxxBaseSpecifier(hasType(
-                                              asString("class X")))
+Exmaple
  class X {};
  void y(X &amp;x) { x; X z; }
  typedef int U;
  class Y { friend class X; };
  class Z : public virtual X {};
+
+The matcher expr(hasType(cxxRecordDecl(hasName("X"))))
+matches x and z.
+The matcher varDecl(hasType(cxxRecordDecl(hasName("X"))))
+matches z
+The matcher typedefDecl(hasType(asString("int")))
+matches typedef int U
+The matcher friendDecl(hasType(asString("class X")))
+matches friend class X
+The matcher cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+asString("X"))).bind("b"))) matches class Z : public virtual X {},
+with cxxBaseSpecifier(...)
+matching public virtual X.
 </pre></td></tr>
 
 
@@ -10502,9 +13876,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasInitializer0"><pre>Matches a variable declaration that has an initializer expression
 that matches the given matcher.
 
-Example matches x (matcher = varDecl(hasInitializer(callExpr())))
-  bool y() { return true; }
-  bool x = y();
+Given
+  int y() { return 0; }
+  void foo() {
+    int x = y();
+  }
+The matcher varDecl(hasInitializer(callExpr()))
+matches x
 </pre></td></tr>
 
 
@@ -10524,13 +13902,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template &lt;typename T, typename U&gt;
   void f(T&amp;&amp; t, U&amp;&amp; u) {}
 
-  bool B = false;
-  f(R, B);
-templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-  matches twice, with expr() matching 'R * 2' and 'R * 4'
-functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-  matches the specialization f&lt;unsigned, bool&gt; twice, for 'unsigned'
-  and 'bool'
+  void foo() {
+    bool B = false;
+    f(R, B);
+  }
+
+The matcher
+templateSpecializationType(forEachTemplateArgument(isExpr(expr().bind("t_arg"))))
+matches Matrix&lt;int, R * 2, R * 4&gt; twice, with
+expr() matching R * 2 and
+R * 4.
+The matcher
+functionDecl(forEachTemplateArgument(refersToType(qualType().bind("type"))))
+matches the specialization of f twice,
+with qualType() matching
+unsigned and
+bool.
 </pre></td></tr>
 
 
@@ -10543,9 +13930,11 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 Given
   template&lt;typename T&gt; class A {};
   A&lt;int&gt; a;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-  hasTypeLoc(loc(asString("int")))))))
-  matches `A&lt;int&gt; a`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+hasTypeLoc(loc(asString("int"))))))))) matches A&lt;int&gt; a.
 </pre></td></tr>
 
 
@@ -10559,15 +13948,19 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;&gt; class A&lt;double&gt; {};
   A&lt;int&gt; a;
 
-  template&lt;typename T&gt; f() {};
+  template&lt;typename T&gt; void f() {};
   void func() { f&lt;int&gt;(); };
 
-classTemplateSpecializationDecl(hasAnyTemplateArgument(
-    refersToType(asString("int"))))
-  matches the specialization A&lt;int&gt;
 
-functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher classTemplateSpecializationDecl(
+                        hasAnyTemplateArgument(
+                          refersToType(asString("int"))))
+matches class A&lt;int&gt;.
+
+The matcher
+functionDecl(hasAnyTemplateArgument(
+              refersToType(asString("int"))))
+matches the instantiation of f.
 </pre></td></tr>
 
 
@@ -10580,9 +13973,12 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   template&lt;typename T, typename U&gt; class A {};
   A&lt;double, int&gt; b;
   A&lt;int, double&gt; c;
-varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-  hasTypeLoc(loc(asString("double")))))))
-  matches `A&lt;double, int&gt; b`, but not `A&lt;int, double&gt; c`.
+
+The matcher
+varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+hasTypeLoc(loc(asString("double")))))))))
+matches A&lt;double, int&gt; b, but not double&gt; c}.
 </pre></td></tr>
 
 
@@ -10593,17 +13989,20 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 
 Given
   template&lt;typename T, typename U&gt; class A {};
-  A&lt;bool, int&gt; b;
-  A&lt;int, bool&gt; c;
+  A&lt;double, int&gt; b;
+  A&lt;int, double&gt; c;
 
   template&lt;typename T&gt; void f() {}
   void func() { f&lt;int&gt;(); };
+
+The matcher
 classTemplateSpecializationDecl(hasTemplateArgument(
     1, refersToType(asString("int"))))
-  matches the specialization A&lt;bool, int&gt;
+matches the specialization class A&lt;double, int&gt;.
 
-functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-  matches the specialization f&lt;int&gt;
+The matcher functionDecl(hasTemplateArgument(0,
+                        refersToType(asString("int"))))
+matches the specialization of f.
 </pre></td></tr>
 
 
@@ -10615,9 +14014,10 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
   void f(int b) {
     int a[b];
   }
+The matcher
 variableArrayType(hasSizeExpr(ignoringImpCasts(declRefExpr(to(
   varDecl(hasName("b")))))))
-  matches "int a[b]"
+matches int[b]
 </pre></td></tr>
 
 
@@ -10628,20 +14028,22 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 other declarations of the same function or coroutine.
 
 Given
+void foo() {
   for (;;) {}
-forStmt(hasBody(compoundStmt()))
-  matches 'for (;;) {}'
+}
+The matcher forStmt(hasBody(compoundStmt().bind("body")))
+matches for (;;) {}
 with compoundStmt()
-  matching '{}'
+  matching {}
 
 Given
   void f();
   void f() {}
-functionDecl(hasBody(compoundStmt()))
-  matches 'void f() {}'
+The matcher functionDecl(hasBody(compoundStmt().bind("compound")))
+f
 with compoundStmt()
-  matching '{}'
-  but does not match 'void f();'
+matching {}
+but does not match void f();
 </pre></td></tr>
 
 
@@ -10649,8 +14051,13 @@ <h2 id="traversal-matchers">AST Traversal Matchers</h2>
 <tr><td colspan="4" class="doc" id="hasCondition2"><pre>Matches the condition expression of an if statement, for loop,
 switch statement or conditional operator.
 
-Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+Given
+void foo() {
   if (true) {}
+}
+
+The matcher ifStmt(hasCondition(cxxBoolLiteral(equals(true))))
+if (true) {}
 </pre></td></tr>
 
 <!--END_TRAVERSAL_MATCHERS -->
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 14907e7db18de..a9cd68d392c75 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -81,6 +81,24 @@ C++ Specific Potentially Breaking Changes
     template <typename T>
     void f();
 
+- During constant evaluation, comparisons between different evaluations of the
+  same string literal are now correctly treated as non-constant, and comparisons
+  between string literals that cannot possibly overlap in memory are now treated
+  as constant. This updates Clang to match the anticipated direction of open core
+  issue `CWG2765 <http://wg21.link/CWG2765>`, but is subject to change once that
+  issue is resolved.
+
+  .. code-block:: c++
+
+    constexpr const char *f() { return "hello"; }
+    constexpr const char *g() { return "world"; }
+    // Used to evaluate to false, now error: non-constant comparison.
+    constexpr bool a = f() == f();
+    // Might evaluate to true or false, as before.
+    bool at_runtime() { return f() == f(); }
+    // Was error, now evaluates to false.
+    constexpr bool b = f() == g();
+
 ABI Changes in This Version
 ---------------------------
 
@@ -550,6 +568,9 @@ AST Matchers
 
 - Fixed a crash when traverse lambda expr with invalid captures. (#GH106444)
 
+- The examples in the AST matcher reference are now tested and additional
+  examples and descriptions were added.
+
 clang-format
 ------------
 
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index 57d78f867bab6..4f03388bc87bd 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2369,14 +2369,16 @@ are listed below.
      $ cd $P/bar && clang -c -funique-internal-linkage-names name_conflict.c
      $ cd $P && clang foo/name_conflict.o && bar/name_conflict.o
 
-.. option:: -fbasic-block-sections=[labels, all, list=<arg>, none]
+.. option:: -f[no]-basic-block-address-map:
+  Emits a ``SHT_LLVM_BB_ADDR_MAP`` section which includes address offsets for each
+  basic block in the program, relative to the parent function address.
+
+
+.. option:: -fbasic-block-sections=[all, list=<arg>, none]
 
   Controls how Clang emits text sections for basic blocks. With values ``all``
   and ``list=<arg>``, each basic block or a subset of basic blocks can be placed
-  in its own unique section. With the "labels" value, normal text sections are
-  emitted, but a ``.bb_addr_map`` section is emitted which includes address
-  offsets for each basic block in the program, relative to the parent function
-  address.
+  in its own unique section.
 
   With the ``list=<arg>`` option, a file containing the subset of basic blocks
   that need to placed in unique sections can be specified.  The format of the
diff --git a/clang/docs/analyzer/checkers.rst b/clang/docs/analyzer/checkers.rst
index c124fefc78611..9847d449d76d0 100644
--- a/clang/docs/analyzer/checkers.rst
+++ b/clang/docs/analyzer/checkers.rst
@@ -1008,6 +1008,241 @@ optin.portability.UnixAPI
 """""""""""""""""""""""""
 Finds implementation-defined behavior in UNIX/Posix functions.
 
+
+optin.taint
+^^^^^^^^^^^
+
+Checkers implementing
+`taint analysis <https://en.wikipedia.org/wiki/Taint_checking>`_.
+
+.. _optin-taint-GenericTaint:
+
+optin.taint.GenericTaint (C, C++)
+"""""""""""""""""""""""""""""""""
+
+Taint analysis identifies potential security vulnerabilities where the
+attacker can inject malicious data to the program to execute an attack
+(privilege escalation, command injection, SQL injection etc.).
+
+The malicious data is injected at the taint source (e.g. ``getenv()`` call)
+which is then propagated through function calls and being used as arguments of
+sensitive operations, also called as taint sinks (e.g. ``system()`` call).
+
+One can defend against this type of vulnerability by always checking and
+sanitizing the potentially malicious, untrusted user input.
+
+The goal of the checker is to discover and show to the user these potential
+taint source-sink pairs and the propagation call chain.
+
+The most notable examples of taint sources are:
+
+  - data from network
+  - files or standard input
+  - environment variables
+  - data from databases
+
+Let us examine a practical example of a Command Injection attack.
+
+.. code-block:: c
+
+  // Command Injection Vulnerability Example
+  int main(int argc, char** argv) {
+    char cmd[2048] = "/bin/cat ";
+    char filename[1024];
+    printf("Filename:");
+    scanf (" %1023[^\n]", filename); // The attacker can inject a shell escape here
+    strcat(cmd, filename);
+    system(cmd); // Warning: Untrusted data is passed to a system call
+  }
+
+The program prints the content of any user specified file.
+Unfortunately the attacker can execute arbitrary commands
+with shell escapes. For example with the following input the `ls` command is also
+executed after the contents of `/etc/shadow` is printed.
+`Input: /etc/shadow ; ls /`
+
+The analysis implemented in this checker points out this problem.
+
+One can protect against such attack by for example checking if the provided
+input refers to a valid file and removing any invalid user input.
+
+.. code-block:: c
+
+  // No vulnerability anymore, but we still get the warning
+  void sanitizeFileName(char* filename){
+    if (access(filename,F_OK)){// Verifying user input
+      printf("File does not exist\n");
+      filename[0]='\0';
+      }
+  }
+  int main(int argc, char** argv) {
+    char cmd[2048] = "/bin/cat ";
+    char filename[1024];
+    printf("Filename:");
+    scanf (" %1023[^\n]", filename); // The attacker can inject a shell escape here
+    sanitizeFileName(filename);// filename is safe after this point
+    if (!filename[0])
+      return -1;
+    strcat(cmd, filename);
+    system(cmd); // Superfluous Warning: Untrusted data is passed to a system call
+  }
+
+Unfortunately, the checker cannot discover automatically that the programmer
+have performed data sanitation, so it still emits the warning.
+
+One can get rid of this superfluous warning by telling by specifying the
+sanitation functions in the taint configuration file (see
+:doc:`user-docs/TaintAnalysisConfiguration`).
+
+.. code-block:: YAML
+
+  Filters:
+  - Name: sanitizeFileName
+    Args: [0]
+
+The clang invocation to pass the configuration file location:
+
+.. code-block:: bash
+
+  clang  --analyze -Xclang -analyzer-config  -Xclang optin.taint.TaintPropagation:Config=`pwd`/taint_config.yml ...
+
+If you are validating your inputs instead of sanitizing them, or don't want to
+mention each sanitizing function in our configuration,
+you can use a more generic approach.
+
+Introduce a generic no-op `csa_mark_sanitized(..)` function to
+tell the Clang Static Analyzer
+that the variable is safe to be used on that analysis path.
+
+.. code-block:: c
+
+  // Marking sanitized variables safe.
+  // No vulnerability anymore, no warning.
+
+  // User csa_mark_sanitize function is for the analyzer only
+  #ifdef __clang_analyzer__
+    void csa_mark_sanitized(const void *);
+  #endif
+
+  int main(int argc, char** argv) {
+    char cmd[2048] = "/bin/cat ";
+    char filename[1024];
+    printf("Filename:");
+    scanf (" %1023[^\n]", filename);
+    if (access(filename,F_OK)){// Verifying user input
+      printf("File does not exist\n");
+      return -1;
+    }
+    #ifdef __clang_analyzer__
+      csa_mark_sanitized(filename); // Indicating to CSA that filename variable is safe to be used after this point
+    #endif
+    strcat(cmd, filename);
+    system(cmd); // No warning
+  }
+
+Similarly to the previous example, you need to
+define a `Filter` function in a `YAML` configuration file
+and add the `csa_mark_sanitized` function.
+
+.. code-block:: YAML
+
+  Filters:
+  - Name: csa_mark_sanitized
+    Args: [0]
+
+Then calling `csa_mark_sanitized(X)` will tell the analyzer that `X` is safe to
+be used after this point, because its contents are verified. It is the
+responsibility of the programmer to ensure that this verification was indeed
+correct. Please note that `csa_mark_sanitized` function is only declared and
+used during Clang Static Analysis and skipped in (production) builds.
+
+Further examples of injection vulnerabilities this checker can find.
+
+.. code-block:: c
+
+  void test() {
+    char x = getchar(); // 'x' marked as tainted
+    system(&x); // warn: untrusted data is passed to a system call
+  }
+
+  // note: compiler internally checks if the second param to
+  // sprintf is a string literal or not.
+  // Use -Wno-format-security to suppress compiler warning.
+  void test() {
+    char s[10], buf[10];
+    fscanf(stdin, "%s", s); // 's' marked as tainted
+
+    sprintf(buf, s); // warn: untrusted data used as a format string
+  }
+
+There are built-in sources, propagations and sinks even if no external taint
+configuration is provided.
+
+Default sources:
+ ``_IO_getc``, ``fdopen``, ``fopen``, ``freopen``, ``get_current_dir_name``,
+ ``getch``, ``getchar``, ``getchar_unlocked``, ``getwd``, ``getcwd``,
+ ``getgroups``, ``gethostname``, ``getlogin``, ``getlogin_r``, ``getnameinfo``,
+ ``gets``, ``gets_s``, ``getseuserbyname``, ``readlink``, ``readlinkat``,
+ ``scanf``, ``scanf_s``, ``socket``, ``wgetch``
+
+Default propagations rules:
+ ``atoi``, ``atol``, ``atoll``, ``basename``, ``dirname``, ``fgetc``,
+ ``fgetln``, ``fgets``, ``fnmatch``, ``fread``, ``fscanf``, ``fscanf_s``,
+ ``index``, ``inflate``, ``isalnum``, ``isalpha``, ``isascii``, ``isblank``,
+ ``iscntrl``, ``isdigit``, ``isgraph``, ``islower``, ``isprint``, ``ispunct``,
+ ``isspace``, ``isupper``, ``isxdigit``, ``memchr``, ``memrchr``, ``sscanf``,
+ ``getc``, ``getc_unlocked``, ``getdelim``, ``getline``, ``getw``, ``memcmp``,
+ ``memcpy``, ``memmem``, ``memmove``, ``mbtowc``, ``pread``, ``qsort``,
+ ``qsort_r``, ``rawmemchr``, ``read``, ``recv``, ``recvfrom``, ``rindex``,
+ ``strcasestr``, ``strchr``, ``strchrnul``, ``strcasecmp``, ``strcmp``,
+ ``strcspn``, ``strncasecmp``, ``strncmp``, ``strndup``,
+ ``strndupa``, ``strpbrk``, ``strrchr``, ``strsep``, ``strspn``,
+ ``strstr``, ``strtol``, ``strtoll``, ``strtoul``, ``strtoull``, ``tolower``,
+ ``toupper``, ``ttyname``, ``ttyname_r``, ``wctomb``, ``wcwidth``
+
+Default sinks:
+ ``printf``, ``setproctitle``, ``system``, ``popen``, ``execl``, ``execle``,
+ ``execlp``, ``execv``, ``execvp``, ``execvP``, ``execve``, ``dlopen``
+
+Please note that there are no built-in filter functions.
+
+One can configure their own taint sources, sinks, and propagation rules by
+providing a configuration file via checker option
+``optin.taint.TaintPropagation:Config``. The configuration file is in
+`YAML <http://llvm.org/docs/YamlIO.html#introduction-to-yaml>`_ format. The
+taint-related options defined in the config file extend but do not override the
+built-in sources, rules, sinks. The format of the external taint configuration
+file is not stable, and could change without any notice even in a non-backward
+compatible way.
+
+For a more detailed description of configuration options, please see the
+:doc:`user-docs/TaintAnalysisConfiguration`. For an example see
+:ref:`clangsa-taint-configuration-example`.
+
+**Configuration**
+
+* `Config`  Specifies the name of the YAML configuration file. The user can
+  define their own taint sources and sinks.
+
+**Related Guidelines**
+
+* `CWE Data Neutralization Issues
+  <https://cwe.mitre.org/data/definitions/137.html>`_
+* `SEI Cert STR02-C. Sanitize data passed to complex subsystems
+  <https://wiki.sei.cmu.edu/confluence/display/c/STR02-C.+Sanitize+data+passed+to+complex+subsystems>`_
+* `SEI Cert ENV33-C. Do not call system()
+  <https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152177>`_
+* `ENV03-C. Sanitize the environment when invoking external programs
+  <https://wiki.sei.cmu.edu/confluence/display/c/ENV03-C.+Sanitize+the+environment+when+invoking+external+programs>`_
+
+**Limitations**
+
+* The taintedness property is not propagated through function calls which are
+  unknown (or too complex) to the analyzer, unless there is a specific
+  propagation rule built-in to the checker or given in the YAML configuration
+  file. This causes potential true positive findings to be lost.
+
+
 .. _optin-taint-TaintedAlloc:
 
 optin.taint.TaintedAlloc (C, C++)
@@ -1026,7 +1261,7 @@ covers the SEI Cert coding standard rule `INT04-C
 
 You can silence this warning either by bound checking the ``size`` parameter, or
 by explicitly marking the ``size`` parameter as sanitized. See the
-:ref:`alpha-security-taint-GenericTaint` checker for an example.
+:ref:`optin-taint-GenericTaint` checker for an example.
 
 .. code-block:: c
 
@@ -2976,7 +3211,7 @@ Warn about buffer overflows (newer checker).
    buf[0][-1] = 1; // warn
  }
 
- // note: requires alpha.security.taint check turned on.
+ // note: requires optin.taint check turned on.
  void test() {
    char s[] = "abc";
    int x = getchar();
@@ -3009,239 +3244,6 @@ alpha.security.cert
 
 SEI CERT checkers which tries to find errors based on their `C coding rules <https://wiki.sei.cmu.edu/confluence/display/c/2+Rules>`_.
 
-alpha.security.taint
-^^^^^^^^^^^^^^^^^^^^
-
-Checkers implementing
-`taint analysis <https://en.wikipedia.org/wiki/Taint_checking>`_.
-
-.. _alpha-security-taint-GenericTaint:
-
-alpha.security.taint.GenericTaint (C, C++)
-""""""""""""""""""""""""""""""""""""""""""
-
-Taint analysis identifies potential security vulnerabilities where the
-attacker can inject malicious data to the program to execute an attack
-(privilege escalation, command injection, SQL injection etc.).
-
-The malicious data is injected at the taint source (e.g. ``getenv()`` call)
-which is then propagated through function calls and being used as arguments of
-sensitive operations, also called as taint sinks (e.g. ``system()`` call).
-
-One can defend against this type of vulnerability by always checking and
-sanitizing the potentially malicious, untrusted user input.
-
-The goal of the checker is to discover and show to the user these potential
-taint source-sink pairs and the propagation call chain.
-
-The most notable examples of taint sources are:
-
-  - data from network
-  - files or standard input
-  - environment variables
-  - data from databases
-
-Let us examine a practical example of a Command Injection attack.
-
-.. code-block:: c
-
-  // Command Injection Vulnerability Example
-  int main(int argc, char** argv) {
-    char cmd[2048] = "/bin/cat ";
-    char filename[1024];
-    printf("Filename:");
-    scanf (" %1023[^\n]", filename); // The attacker can inject a shell escape here
-    strcat(cmd, filename);
-    system(cmd); // Warning: Untrusted data is passed to a system call
-  }
-
-The program prints the content of any user specified file.
-Unfortunately the attacker can execute arbitrary commands
-with shell escapes. For example with the following input the `ls` command is also
-executed after the contents of `/etc/shadow` is printed.
-`Input: /etc/shadow ; ls /`
-
-The analysis implemented in this checker points out this problem.
-
-One can protect against such attack by for example checking if the provided
-input refers to a valid file and removing any invalid user input.
-
-.. code-block:: c
-
-  // No vulnerability anymore, but we still get the warning
-  void sanitizeFileName(char* filename){
-    if (access(filename,F_OK)){// Verifying user input
-      printf("File does not exist\n");
-      filename[0]='\0';
-      }
-  }
-  int main(int argc, char** argv) {
-    char cmd[2048] = "/bin/cat ";
-    char filename[1024];
-    printf("Filename:");
-    scanf (" %1023[^\n]", filename); // The attacker can inject a shell escape here
-    sanitizeFileName(filename);// filename is safe after this point
-    if (!filename[0])
-      return -1;
-    strcat(cmd, filename);
-    system(cmd); // Superfluous Warning: Untrusted data is passed to a system call
-  }
-
-Unfortunately, the checker cannot discover automatically that the programmer
-have performed data sanitation, so it still emits the warning.
-
-One can get rid of this superfluous warning by telling by specifying the
-sanitation functions in the taint configuration file (see
-:doc:`user-docs/TaintAnalysisConfiguration`).
-
-.. code-block:: YAML
-
-  Filters:
-  - Name: sanitizeFileName
-    Args: [0]
-
-The clang invocation to pass the configuration file location:
-
-.. code-block:: bash
-
-  clang  --analyze -Xclang -analyzer-config  -Xclang alpha.security.taint.TaintPropagation:Config=`pwd`/taint_config.yml ...
-
-If you are validating your inputs instead of sanitizing them, or don't want to
-mention each sanitizing function in our configuration,
-you can use a more generic approach.
-
-Introduce a generic no-op `csa_mark_sanitized(..)` function to
-tell the Clang Static Analyzer
-that the variable is safe to be used on that analysis path.
-
-.. code-block:: c
-
-  // Marking sanitized variables safe.
-  // No vulnerability anymore, no warning.
-
-  // User csa_mark_sanitize function is for the analyzer only
-  #ifdef __clang_analyzer__
-    void csa_mark_sanitized(const void *);
-  #endif
-
-  int main(int argc, char** argv) {
-    char cmd[2048] = "/bin/cat ";
-    char filename[1024];
-    printf("Filename:");
-    scanf (" %1023[^\n]", filename);
-    if (access(filename,F_OK)){// Verifying user input
-      printf("File does not exist\n");
-      return -1;
-    }
-    #ifdef __clang_analyzer__
-      csa_mark_sanitized(filename); // Indicating to CSA that filename variable is safe to be used after this point
-    #endif
-    strcat(cmd, filename);
-    system(cmd); // No warning
-  }
-
-Similarly to the previous example, you need to
-define a `Filter` function in a `YAML` configuration file
-and add the `csa_mark_sanitized` function.
-
-.. code-block:: YAML
-
-  Filters:
-  - Name: csa_mark_sanitized
-    Args: [0]
-
-Then calling `csa_mark_sanitized(X)` will tell the analyzer that `X` is safe to
-be used after this point, because its contents are verified. It is the
-responsibility of the programmer to ensure that this verification was indeed
-correct. Please note that `csa_mark_sanitized` function is only declared and
-used during Clang Static Analysis and skipped in (production) builds.
-
-Further examples of injection vulnerabilities this checker can find.
-
-.. code-block:: c
-
-  void test() {
-    char x = getchar(); // 'x' marked as tainted
-    system(&x); // warn: untrusted data is passed to a system call
-  }
-
-  // note: compiler internally checks if the second param to
-  // sprintf is a string literal or not.
-  // Use -Wno-format-security to suppress compiler warning.
-  void test() {
-    char s[10], buf[10];
-    fscanf(stdin, "%s", s); // 's' marked as tainted
-
-    sprintf(buf, s); // warn: untrusted data used as a format string
-  }
-
-There are built-in sources, propagations and sinks even if no external taint
-configuration is provided.
-
-Default sources:
- ``_IO_getc``, ``fdopen``, ``fopen``, ``freopen``, ``get_current_dir_name``,
- ``getch``, ``getchar``, ``getchar_unlocked``, ``getwd``, ``getcwd``,
- ``getgroups``, ``gethostname``, ``getlogin``, ``getlogin_r``, ``getnameinfo``,
- ``gets``, ``gets_s``, ``getseuserbyname``, ``readlink``, ``readlinkat``,
- ``scanf``, ``scanf_s``, ``socket``, ``wgetch``
-
-Default propagations rules:
- ``atoi``, ``atol``, ``atoll``, ``basename``, ``dirname``, ``fgetc``,
- ``fgetln``, ``fgets``, ``fnmatch``, ``fread``, ``fscanf``, ``fscanf_s``,
- ``index``, ``inflate``, ``isalnum``, ``isalpha``, ``isascii``, ``isblank``,
- ``iscntrl``, ``isdigit``, ``isgraph``, ``islower``, ``isprint``, ``ispunct``,
- ``isspace``, ``isupper``, ``isxdigit``, ``memchr``, ``memrchr``, ``sscanf``,
- ``getc``, ``getc_unlocked``, ``getdelim``, ``getline``, ``getw``, ``memcmp``,
- ``memcpy``, ``memmem``, ``memmove``, ``mbtowc``, ``pread``, ``qsort``,
- ``qsort_r``, ``rawmemchr``, ``read``, ``recv``, ``recvfrom``, ``rindex``,
- ``strcasestr``, ``strchr``, ``strchrnul``, ``strcasecmp``, ``strcmp``,
- ``strcspn``, ``strncasecmp``, ``strncmp``, ``strndup``,
- ``strndupa``, ``strpbrk``, ``strrchr``, ``strsep``, ``strspn``,
- ``strstr``, ``strtol``, ``strtoll``, ``strtoul``, ``strtoull``, ``tolower``,
- ``toupper``, ``ttyname``, ``ttyname_r``, ``wctomb``, ``wcwidth``
-
-Default sinks:
- ``printf``, ``setproctitle``, ``system``, ``popen``, ``execl``, ``execle``,
- ``execlp``, ``execv``, ``execvp``, ``execvP``, ``execve``, ``dlopen``
-
-Please note that there are no built-in filter functions.
-
-One can configure their own taint sources, sinks, and propagation rules by
-providing a configuration file via checker option
-``alpha.security.taint.TaintPropagation:Config``. The configuration file is in
-`YAML <http://llvm.org/docs/YamlIO.html#introduction-to-yaml>`_ format. The
-taint-related options defined in the config file extend but do not override the
-built-in sources, rules, sinks. The format of the external taint configuration
-file is not stable, and could change without any notice even in a non-backward
-compatible way.
-
-For a more detailed description of configuration options, please see the
-:doc:`user-docs/TaintAnalysisConfiguration`. For an example see
-:ref:`clangsa-taint-configuration-example`.
-
-**Configuration**
-
-* `Config`  Specifies the name of the YAML configuration file. The user can
-  define their own taint sources and sinks.
-
-**Related Guidelines**
-
-* `CWE Data Neutralization Issues
-  <https://cwe.mitre.org/data/definitions/137.html>`_
-* `SEI Cert STR02-C. Sanitize data passed to complex subsystems
-  <https://wiki.sei.cmu.edu/confluence/display/c/STR02-C.+Sanitize+data+passed+to+complex+subsystems>`_
-* `SEI Cert ENV33-C. Do not call system()
-  <https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152177>`_
-* `ENV03-C. Sanitize the environment when invoking external programs
-  <https://wiki.sei.cmu.edu/confluence/display/c/ENV03-C.+Sanitize+the+environment+when+invoking+external+programs>`_
-
-**Limitations**
-
-* The taintedness property is not propagated through function calls which are
-  unknown (or too complex) to the analyzer, unless there is a specific
-  propagation rule built-in to the checker or given in the YAML configuration
-  file. This causes potential true positive findings to be lost.
-
 alpha.unix
 ^^^^^^^^^^
 
@@ -3440,6 +3442,27 @@ Check for non-determinism caused by sorting of pointers.
 alpha.WebKit
 ^^^^^^^^^^^^
 
+.. _alpha-webkit-NoUncheckedPtrMemberChecker:
+
+alpha.webkit.NoUncheckedPtrMemberChecker
+""""""""""""""""""""""""""""""""""""""""
+Raw pointers and references to an object which supports CheckedPtr or CheckedRef can't be used as class members. Only CheckedPtr, CheckedRef, RefPtr, or Ref are allowed.
+
+.. code-block:: cpp
+
+ struct CheckableObj {
+   void incrementPtrCount() {}
+   void decrementPtrCount() {}
+ };
+
+ struct Foo {
+   CheckableObj* ptr; // warn
+   CheckableObj& ptr; // warn
+   // ...
+ };
+
+See `WebKit Guidelines for Safer C++ Programming <https://github.com/WebKit/WebKit/wiki/Safer-CPP-Guidelines>`_ for details.
+
 .. _alpha-webkit-UncountedCallArgsChecker:
 
 alpha.webkit.UncountedCallArgsChecker
diff --git a/clang/docs/analyzer/user-docs/TaintAnalysisConfiguration.rst b/clang/docs/analyzer/user-docs/TaintAnalysisConfiguration.rst
index 67e71d558f2ce..4849a553cb2ce 100644
--- a/clang/docs/analyzer/user-docs/TaintAnalysisConfiguration.rst
+++ b/clang/docs/analyzer/user-docs/TaintAnalysisConfiguration.rst
@@ -4,10 +4,10 @@ Taint Analysis Configuration
 
 The Clang Static Analyzer uses taint analysis to detect injection vulnerability related issues in code.
 The backbone of taint analysis in the Clang SA is the ``TaintPropagation`` modeling checker.
-The reports are emitted via the :ref:`alpha-security-taint-GenericTaint` checker.
+The reports are emitted via the :ref:`optin-taint-GenericTaint` checker.
 The ``TaintPropagation`` checker has a default taint-related configuration.
 The built-in default settings are defined in code, and they are always in effect.
-The checker also provides a configuration interface for extending the default settings via the ``alpha.security.taint.TaintPropagation:Config`` checker config parameter
+The checker also provides a configuration interface for extending the default settings via the ``optin.taint.TaintPropagation:Config`` checker config parameter
 by providing a configuration file to the in `YAML <http://llvm.org/docs/YamlIO.html#introduction-to-yaml>`_ format.
 This documentation describes the syntax of the configuration file and gives the informal semantics of the configuration options.
 
diff --git a/clang/docs/doxygen.cfg.in b/clang/docs/doxygen.cfg.in
index 251afb179b205..1d1deb0fcfb07 100644
--- a/clang/docs/doxygen.cfg.in
+++ b/clang/docs/doxygen.cfg.in
@@ -220,7 +220,14 @@ TAB_SIZE               = 2
 # "Side Effects:". You can put \n's in the value part of an alias to insert
 # newlines.
 
-ALIASES                =
+ALIASES                += compile_args{1}="Compiled with <tt>\1</tt>.\n"
+ALIASES                += matcher{1}="<tt>\1</tt>"
+ALIASES                += matcher{2$}="<tt>\2</tt>"
+ALIASES                += match{1}="<tt>\1</tt>"
+ALIASES                += match{2$}="<tt>\2</tt>"
+ALIASES                += nomatch{1}="<tt>\1</tt>"
+ALIASES                += header{1}="\code"
+ALIASES                += endheader="\endcode"
 
 # This tag can be used to specify a number of word-keyword mappings (TCL only).
 # A mapping has the form "name=value". For example adding "class=itcl::class"
diff --git a/clang/docs/tools/dump_ast_matchers.py b/clang/docs/tools/dump_ast_matchers.py
index 705ff0d4d4098..a3feac8728c65 100755
--- a/clang/docs/tools/dump_ast_matchers.py
+++ b/clang/docs/tools/dump_ast_matchers.py
@@ -100,15 +100,72 @@ def extract_result_types(comment):
         comment = m.group(1)
 
 
+def find_next_closing_rbrace(
+    data: str, start_pos: int, braces_to_be_matched: int
+) -> int:
+    """Finds the location of the closing rbrace '}' inside of data."""
+    """'start_pos' should be one past the opening lbrace and braces_to_be_matched is initialized with 0"""
+    next_lbrace = data.find("{", start_pos)
+    next_rbrace = data.find("}", start_pos)
+    if next_lbrace != -1:
+        if next_lbrace < next_rbrace:
+            return find_next_closing_rbrace(
+                data, next_lbrace + 1, braces_to_be_matched + 1
+            )
+        if braces_to_be_matched == 0:
+            return next_rbrace
+        return find_next_closing_rbrace(data, next_rbrace + 1, braces_to_be_matched - 1)
+
+    if braces_to_be_matched > 0:
+        return find_next_closing_rbrace(data, next_rbrace + 1, braces_to_be_matched - 1)
+
+    return next_rbrace
+
+
 def strip_doxygen(comment):
     """Returns the given comment without \-escaped words."""
-    # If there is only a doxygen keyword in the line, delete the whole line.
-    comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)
-
     # If there is a doxygen \see command, change the \see prefix into "See also:".
     # FIXME: it would be better to turn this into a link to the target instead.
     comment = re.sub(r"\\see", r"See also:", comment)
 
+    commands: list[str] = [
+        "\\compile_args{",
+        "\\matcher{",
+        "\\match{",
+        "\\nomatch{",
+    ]
+
+    for command in commands:
+        delete_command = command == "\\compile_args{"
+        command_begin_loc = comment.find(command)
+        while command_begin_loc != -1:
+            command_end_loc = command_begin_loc + len(command)
+            end_brace_loc = find_next_closing_rbrace(comment, command_end_loc + 1, 0)
+            if end_brace_loc == -1:
+                print("found unmatched {")
+                command_begin_loc = comment.find(command, command_end_loc)
+                continue
+
+            if delete_command:
+                comment = comment[0:command_begin_loc] + comment[end_brace_loc + 1 :]
+                command_begin_loc = comment.find(command, command_begin_loc)
+                continue
+
+            tag_seperator_loc = comment.find("$", command_end_loc)
+            if tag_seperator_loc != -1 and tag_seperator_loc < end_brace_loc:
+                command_end_loc = tag_seperator_loc + 1
+
+            comment = (
+                comment[0:command_begin_loc]
+                + comment[command_end_loc:end_brace_loc]
+                + comment[end_brace_loc + 1 :]
+            )
+
+            command_begin_loc = comment.find(command, command_begin_loc)
+
+    # If there is only a doxygen keyword in the line, delete the whole line.
+    comment = re.sub(r"^\\[^\s]+\n", r"", comment, flags=re.M)
+
     # Delete the doxygen command and the following whitespace.
     comment = re.sub(r"\\[^\s]+\s+", r"", comment)
     return comment
@@ -191,8 +248,9 @@ def act_on_decl(declaration, comment, allowed_types):
     definition.
     """
     if declaration.strip():
-
-        if re.match(r"^\s?(#|namespace|using|template <typename NodeType> using|})", declaration):
+        if re.match(
+            r"^\s?(#|namespace|using|template <typename NodeType> using|})", declaration
+        ):
             return
 
         # Node matchers are defined by writing:
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 1984310df0442..3db9871a4b07b 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -253,7 +253,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mutable llvm::FoldingSet<BitIntType> BitIntTypes;
   mutable llvm::ContextualFoldingSet<DependentBitIntType, ASTContext &>
       DependentBitIntTypes;
-  llvm::FoldingSet<BTFTagAttributedType> BTFTagAttributedTypes;
+  mutable llvm::FoldingSet<BTFTagAttributedType> BTFTagAttributedTypes;
   llvm::FoldingSet<HLSLAttributedResourceType> HLSLAttributedResourceTypes;
 
   mutable llvm::FoldingSet<CountAttributedType> CountAttributedTypes;
@@ -324,6 +324,14 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// This is lazily created.  This is intentionally not serialized.
   mutable llvm::StringMap<StringLiteral *> StringLiteralCache;
 
+  /// The next string literal "version" to allocate during constant evaluation.
+  /// This is used to distinguish between repeated evaluations of the same
+  /// string literal.
+  ///
+  /// We don't need to serialize this because constants get re-evaluated in the
+  /// current file before they are compared locally.
+  unsigned NextStringLiteralVersion = 0;
+
   /// MD5 hash of CUID. It is calculated when first used and cached by this
   /// data member.
   mutable std::string CUIDHash;
@@ -1369,10 +1377,21 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// calling T.withConst().
   QualType getConstType(QualType T) const { return T.withConst(); }
 
+  /// Rebuild a type, preserving any existing type sugar. For function types,
+  /// you probably want to just use \c adjustFunctionResultType and friends
+  /// instead.
+  QualType adjustType(QualType OldType,
+                      llvm::function_ref<QualType(QualType)> Adjust) const;
+
   /// Change the ExtInfo on a function type.
   const FunctionType *adjustFunctionType(const FunctionType *Fn,
                                          FunctionType::ExtInfo EInfo);
 
+  /// Change the result type of a function type, preserving sugar such as
+  /// attributed types.
+  QualType adjustFunctionResultType(QualType FunctionType,
+                                    QualType NewResultType);
+
   /// Adjust the given function result type.
   CanQualType getCanonicalFunctionResultType(QualType ResultType) const;
 
@@ -1702,7 +1721,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
                              QualType equivalentType) const;
 
   QualType getBTFTagAttributedType(const BTFTypeTagAttr *BTFAttr,
-                                   QualType Wrapped);
+                                   QualType Wrapped) const;
 
   QualType getHLSLAttributedResourceType(
       QualType Wrapped, QualType Contained,
@@ -3289,6 +3308,10 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// PredefinedExpr to cache evaluated results.
   StringLiteral *getPredefinedStringLiteralFromCache(StringRef Key) const;
 
+  /// Return the next version number to be used for a string literal evaluated
+  /// as part of constant evaluation.
+  unsigned getNextStringLiteralVersion() { return NextStringLiteralVersion++; }
+
   /// Return a declaration for the global GUID object representing the given
   /// GUID value.
   MSGuidDecl *getMSGuidDecl(MSGuidDeclParts Parts) const;
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 0600ecc4d14a1..7ff35d73df599 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -133,6 +133,10 @@ class TranslationUnitDecl : public Decl,
   static TranslationUnitDecl *castFromDeclContext(const DeclContext *DC) {
     return static_cast<TranslationUnitDecl *>(const_cast<DeclContext*>(DC));
   }
+
+  /// Retrieves the canonical declaration of this translation unit.
+  TranslationUnitDecl *getCanonicalDecl() override { return getFirstDecl(); }
+  const TranslationUnitDecl *getCanonicalDecl() const { return getFirstDecl(); }
 };
 
 /// Represents a `#pragma comment` line. Always a child of
diff --git a/clang/include/clang/AST/DeclID.h b/clang/include/clang/AST/DeclID.h
index f4607e42c4be3..49964b43c7d1d 100644
--- a/clang/include/clang/AST/DeclID.h
+++ b/clang/include/clang/AST/DeclID.h
@@ -189,6 +189,7 @@ class LocalDeclID : public DeclIDBase {
   // Every Decl ID is a local decl ID to the module being writing in ASTWriter.
   friend class ASTWriter;
   friend class GlobalDeclID;
+  friend struct llvm::DenseMapInfo<clang::LocalDeclID>;
 
 public:
   LocalDeclID() : Base() {}
@@ -267,6 +268,27 @@ template <> struct DenseMapInfo<clang::GlobalDeclID> {
   }
 };
 
+template <> struct DenseMapInfo<clang::LocalDeclID> {
+  using LocalDeclID = clang::LocalDeclID;
+  using DeclID = LocalDeclID::DeclID;
+
+  static LocalDeclID getEmptyKey() {
+    return LocalDeclID(DenseMapInfo<DeclID>::getEmptyKey());
+  }
+
+  static LocalDeclID getTombstoneKey() {
+    return LocalDeclID(DenseMapInfo<DeclID>::getTombstoneKey());
+  }
+
+  static unsigned getHashValue(const LocalDeclID &Key) {
+    return DenseMapInfo<DeclID>::getHashValue(Key.getRawValue());
+  }
+
+  static bool isEqual(const LocalDeclID &L, const LocalDeclID &R) {
+    return L == R;
+  }
+};
+
 } // namespace llvm
 
 #endif
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index f1c72efc23878..2dc96184a0460 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -39,6 +39,158 @@
 //  See ASTMatchFinder.h for how to use the generated matchers to run over
 //  an AST.
 //
+//  The doxygen comments on matchers are used to:
+//   - create the doxygen documentation
+//   - get information in the editor via signature help and goto definition
+//   - generate the AST matcher reference html file
+//   - test the documentation using a special syntax
+//
+//  Test Annotations:
+//
+//  The automatic testing uses doxygen commands (aliases) to extract the
+//  relevant information about an example of using a matcher from the
+//  documentation.
+//
+//      \header{a.h}
+//      \endheader     <- zero or more header
+//
+//      \code
+//        int a = 42;
+//      \endcode
+//      \compile_args{-std=c++,c23-or-later} <- optional, the std flag supports
+//      std ranges and
+//                                              whole languages
+//
+//      \matcher{expr()}              <- one or more matchers in succession
+//      \matcher{integerLiteral()}    <- one or more matchers in succession
+//                                    both matcher will have to match the
+//                                    following matches
+//      \match{42}                    <- one or more matches in succession
+//
+//      \matcher{varDecl()} <- new matcher resets the context, the above
+//                             \match will not count for this new
+//                             matcher(-group)
+//      \match{int a  = 42} <- only applies to the previous matcher (not to the
+//                             previous case)
+//
+//
+//  The above block can be repeated inside a doxygen command for multiple code
+//  examples for a single matcher. The test generation script will only look for
+//  these annotations and ignore anything else like `\c` or the sentences where
+//  these annotations are embedded into: `The matcher \matcher{expr()} matches
+//  the number \match{42}.`.
+//
+//  Language Grammar:
+//
+//    [] denotes an optional, and <> denotes user-input
+//
+//    compile_args j:= \compile_args{[<compile_arg>;]<compile_arg>}
+//    matcher_tag_key ::= type
+//    match_tag_key ::= type || std || count || sub
+//    matcher_tags ::= [matcher_tag_key=<value>;]matcher_tag_key=<value>
+//    match_tags ::= [match_tag_key=<value>;]match_tag_key=<value>
+//    matcher ::= \matcher{[matcher_tags$]<matcher>}
+//    matchers ::= [matcher] matcher
+//    match ::= \match{[match_tags$]<match>}
+//    matches ::= [match] match
+//    case ::= matchers matches
+//    cases ::= [case] case
+//    header-block ::= \header{<name>} <code> \endheader
+//    code-block ::= \code <code> \endcode
+//    testcase ::= code-block [compile_args] cases
+//
+//  Language Standard Versions:
+//
+//  The 'std' tag and '\compile_args' support specifying a specific language
+//  version, a whole language and all of its versions, and thresholds (implies
+//  ranges). Multiple arguments are passed with a ',' separator. For a language
+//  and version to execute a tested matcher, it has to match the specified
+//  '\compile_args' for the code, and the 'std' tag for the matcher. Predicates
+//  for the 'std' compiler flag are used with disjunction between languages
+//  (e.g. 'c || c++') and conjunction for all predicates specific to each
+//  language (e.g. 'c++11-or-later && c++23-or-earlier').
+//
+//  Examples:
+//   - `c`                                    all available versions of C
+//   - `c++11`                                only C++11
+//   - `c++11-or-later`                       C++11 or later
+//   - `c++11-or-earlier`                     C++11 or earlier
+//   - `c++11-or-later,c++23-or-earlier,c`    all of C and C++ between 11 and
+//                                            23 (inclusive)
+//   - `c++11-23,c`                             same as above
+//
+//  Tags
+//
+//  `type`:
+//  **Match types** are used to select where the string that is used to check if
+//  a node matches comes from. Available: `code`, `name`, `typestr`,
+//  `typeofstr`. The default is `code`.
+//
+//   - `code`: Forwards to `tooling::fixit::getText(...)` and should be the
+//   preferred way to show what matches.
+//   - `name`: Casts the match to a `NamedDecl` and returns the result of
+//   `getNameAsString`. Useful when the matched AST node is not easy to spell
+//   out (`code` type), e.g., namespaces or classes with many members.
+//   - `typestr`: Returns the result of `QualType::getAsString` for the type
+//   derived from `Type` (otherwise, if it is derived from `Decl`, recurses with
+//   `Node->getTypeForDecl()`)
+//
+//  **Matcher types** are used to mark matchers as sub-matcher with 'sub' or as
+//  deactivated using 'none'. Testing sub-matcher is not implemented.
+//
+//  `count`:
+//  Specifying a 'count=n' on a match will result in a test that requires that
+//  the specified match will be matched n times. Default is 1.
+//
+//  `std`:
+//  A match allows specifying if it matches only in specific language versions.
+//  This may be needed when the AST differs between language versions.
+//
+//  `sub`:
+//  The `sub` tag on a `\match` will indicate that the match is for a node of a
+//  bound sub-matcher. E.g., `\matcher{expr(expr().bind("inner"))}` has a
+//  sub-matcher that binds to `inner`, which is the value for the `sub` tag of
+//  the expected match for the sub-matcher `\match{sub=inner$...}`. Currently,
+//  sub-matchers are not tested in any way.
+//
+//
+//  What if ...?
+//
+//  ... I want to add a matcher?
+//
+//  Add a doxygen comment to the matcher with a code example, corresponding
+//  matchers and matches, that shows what the matcher is supposed to do. Specify
+//  the compile arguments/supported languages if required, and run `ninja
+//  check-clang-unit` to test the documentation.
+//
+//  ... the example I wrote is wrong?
+//
+//  The test-failure output of the generated test file will provide information
+//  about
+//   - where the generated test file is located
+//   - which line in `ASTMatcher.h` the example is from
+//   - which matches were: found, not-(yet)-found, expected
+//   - in case of an unexpected match: what the node looks like using the
+//   different `type`s
+//   - the language version and if the test ran with a windows `-target` flag
+//   (also in failure summary)
+//
+//  ... I don't adhere to the required order of the syntax?
+//
+//  The script will diagnose any found issues, such as `matcher is missing an
+//  example` with a `file:line:` prefix, which should provide enough information
+//  about the issue.
+//
+//  ... the script diagnoses a false-positive issue with a doxygen comment?
+//
+//  It hopefully shouldn't, but if you, e.g., added some non-matcher code and
+//  documented it with doxygen, then the script will consider that as a matcher
+//  documentation. As a result, the script will print that it detected a
+//  mismatch between the actual and the expected number of failures. If the
+//  diagnostic truly is a false-positive, change the
+//  `expected_failure_statistics` at the top of the
+//  `generate_ast_matcher_doc_tests.py` file.
+//
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_ASTMATCHERS_ASTMATCHERS_H
@@ -160,13 +312,13 @@ using AttrMatcher = internal::Matcher<Attr>;
 /// additional constraint. This will often be used with an explicit conversion
 /// to an \c internal::Matcher<> type such as \c TypeMatcher.
 ///
-/// Example: \c DeclarationMatcher(anything()) matches all declarations, e.g.,
+/// Given
 /// \code
-/// "int* p" and "void f()" in
 ///   int* p;
 ///   void f();
 /// \endcode
-///
+/// The matcher \matcher{decl(anything())}
+/// matches \match{int* p} and \match{void f()}.
 /// Usable as: Any Matcher
 inline internal::TrueMatcher anything() { return internal::TrueMatcher(); }
 
@@ -175,12 +327,13 @@ inline internal::TrueMatcher anything() { return internal::TrueMatcher(); }
 /// Given
 /// \code
 ///   int X;
-///   namespace NS {
-///   int Y;
-///   }  // namespace NS
+///   namespace NS { int Y; }
 /// \endcode
-/// decl(hasDeclContext(translationUnitDecl()))
-///   matches "int X", but not "int Y".
+/// \compile_args{-std=c++}
+/// The matcher \matcher{namedDecl(hasDeclContext(translationUnitDecl()))}
+/// matches \match{int X} and \match{namespace NS { int Y; }},
+/// but does not match \nomatch{int Y} because its decl-context is the
+/// namespace \c NS .
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TranslationUnitDecl>
     translationUnitDecl;
 
@@ -191,8 +344,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, TranslationUnitDecl>
 ///   typedef int X;
 ///   using Y = int;
 /// \endcode
-/// typedefDecl()
-///   matches "typedef int X", but not "using Y = int"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{typedefDecl()}
+/// matches \match{typedef int X},
+/// but does not match \nomatch{using Y = int}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TypedefDecl>
     typedefDecl;
 
@@ -203,8 +358,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, TypedefDecl>
 ///   typedef int X;
 ///   using Y = int;
 /// \endcode
-/// typedefNameDecl()
-///   matches "typedef int X" and "using Y = int"
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{typedefNameDecl()}
+/// matches \match{typedef int X} and \match{using Y = int}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TypedefNameDecl>
     typedefNameDecl;
 
@@ -215,34 +371,45 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, TypedefNameDecl>
 ///   typedef int X;
 ///   using Y = int;
 /// \endcode
-/// typeAliasDecl()
-///   matches "using Y = int", but not "typedef int X"
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{typeAliasDecl()}
+/// matches \match{using Y = int},
+/// but does not match \nomatch{typedef int X}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasDecl>
     typeAliasDecl;
 
 /// Matches type alias template declarations.
 ///
-/// typeAliasTemplateDecl() matches
+/// Given
 /// \code
-///   template <typename T>
-///   using Y = X<T>;
+///   template <typename T> struct X {};
+///   template <typename T> using Y = X<T>;
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{typeAliasTemplateDecl()}
+/// matches \match{template <typename T> using Y = X<T>}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasTemplateDecl>
     typeAliasTemplateDecl;
 
 /// Matches AST nodes that were expanded within the main-file.
 ///
-/// Example matches X but not Y
-///   (matcher = cxxRecordDecl(isExpansionInMainFile())
-/// \code
-///   #include <Y.h>
-///   class X {};
-/// \endcode
-/// Y.h:
+/// Given the header \c Y.h
+/// \header{Y.h}
+///   #pragma once
+///   typedef int my_header_int;
+/// \endheader
+/// and the source file
 /// \code
-///   class Y {};
+///   #include "Y.h"
+///   typedef int my_main_file_int;
+///   my_main_file_int a = 0;
+///   my_header_int b = 1;
 /// \endcode
 ///
+/// The matcher \matcher{typedefDecl(isExpansionInMainFile())}
+/// matches \match{typedef int my_main_file_int},
+/// but does not match \nomatch{typedef int my_header_int}.
+///
 /// Usable as: Matcher<Decl>, Matcher<Stmt>, Matcher<TypeLoc>
 AST_POLYMORPHIC_MATCHER(isExpansionInMainFile,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt, TypeLoc)) {
@@ -253,16 +420,21 @@ AST_POLYMORPHIC_MATCHER(isExpansionInMainFile,
 
 /// Matches AST nodes that were expanded within system-header-files.
 ///
-/// Example matches Y but not X
-///     (matcher = cxxRecordDecl(isExpansionInSystemHeader())
+/// Given the header \c SystemHeader.h
+/// \header{system_include/SystemHeader.h}
+///   #pragma once
+///   int header();
+/// \endheader
+/// and the source code
 /// \code
 ///   #include <SystemHeader.h>
-///   class X {};
-/// \endcode
-/// SystemHeader.h:
-/// \code
-///   class Y {};
+///   static int main_file();
 /// \endcode
+/// \compile_args{-isystemsystem_include/}
+///
+/// The matcher \matcher{type=none$functionDecl(isExpansionInSystemHeader())}
+/// matches \match{int header()},
+/// but does not match \nomatch{static int main_file()}.
 ///
 /// Usable as: Matcher<Decl>, Matcher<Stmt>, Matcher<TypeLoc>
 AST_POLYMORPHIC_MATCHER(isExpansionInSystemHeader,
@@ -278,17 +450,32 @@ AST_POLYMORPHIC_MATCHER(isExpansionInSystemHeader,
 /// Matches AST nodes that were expanded within files whose name is
 /// partially matching a given regex.
 ///
-/// Example matches Y but not X
-///     (matcher = cxxRecordDecl(isExpansionInFileMatching("AST.*"))
-/// \code
-///   #include "ASTMatcher.h"
-///   class X {};
-/// \endcode
-/// ASTMatcher.h:
-/// \code
-///   class Y {};
+/// Given the headers \c Y.h
+/// \header{Y.h}
+///   #pragma once
+///   typedef int my_y_int;
+/// \endheader
+/// and \c X.h
+/// \header{X.h}
+///   #pragma once
+///   typedef int my_x_int;
+/// \endheader
+/// and the source code
+/// \code
+///   #include "X.h"
+///   #include "Y.h"
+///   typedef int my_main_file_int;
+///   my_main_file_int a = 0;
+///   my_x_int b = 1;
+///   my_y_int c = 2;
 /// \endcode
 ///
+/// The matcher
+/// \matcher{type=none$typedefDecl(isExpansionInFileMatching("Y.h"))}
+/// matches \match{typedef int my_y_int},
+/// but does not match \nomatch{typedef int my_main_file_int} or
+/// \nomatch{typedef int my_x_int}.
+///
 /// Usable as: Matcher<Decl>, Matcher<Stmt>, Matcher<TypeLoc>
 AST_POLYMORPHIC_MATCHER_REGEX(isExpansionInFileMatching,
                               AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt,
@@ -313,6 +500,17 @@ AST_POLYMORPHIC_MATCHER_REGEX(isExpansionInFileMatching,
 /// Does not match if only part of the statement is expanded from that macro or
 /// if different parts of the statement are expanded from different
 /// appearances of the macro.
+///
+/// Given
+/// \code
+///   #define A 0
+///   #define B A
+///   int c = B;
+/// \endcode
+///
+/// The matcher \matcher{integerLiteral(isExpandedFromMacro("A"))}
+/// matches the literal expanded at the initializer \match{B} of the variable
+/// \c c .
 AST_POLYMORPHIC_MATCHER_P(isExpandedFromMacro,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt, TypeLoc),
                           std::string, MacroName) {
@@ -330,35 +528,50 @@ AST_POLYMORPHIC_MATCHER_P(isExpandedFromMacro,
 
 /// Matches declarations.
 ///
-/// Examples matches \c X, \c C, and the friend declaration inside \c C;
+/// Given
 /// \code
 ///   void X();
 ///   class C {
-///     friend X;
+///     friend void X();
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{decl()}
+/// matches \match{void X()} once, \match{type=name;count=2$C}
+/// twice, once for the definition and once for the implicit class declaration,
+/// and \match{count=2$friend void X()} twice, once for the declaration of the
+/// friend, and once for the redeclaration of the function itself.
 extern const internal::VariadicAllOfMatcher<Decl> decl;
 
 /// Matches decomposition-declarations.
 ///
-/// Examples matches the declaration node with \c foo and \c bar, but not
-/// \c number.
-/// (matcher = declStmt(has(decompositionDecl())))
-///
+/// Given
 /// \code
+///   struct pair { int x; int y; };
+///   pair make(int, int);
 ///   int number = 42;
-///   auto [foo, bar] = std::make_pair{42, 42};
+///   auto [foo, bar] = make(42, 42);
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+/// The matcher \matcher{decompositionDecl()}
+/// matches \match{auto [foo, bar] = make(42, 42)},
+/// but does not match \nomatch{type=name$number}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, DecompositionDecl>
     decompositionDecl;
 
 /// Matches binding declarations
-/// Example matches \c foo and \c bar
-/// (matcher = bindingDecl()
 ///
+/// Given
 /// \code
-///   auto [foo, bar] = std::make_pair{42, 42};
+///   struct pair { int x; int y; };
+///   pair make(int, int);
+///   void f() {
+///     auto [foo, bar] = make(42, 42);
+///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+/// The matcher \matcher{bindingDecl()}
+/// matches \match{type=name$foo} and \match{type=name$bar}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, BindingDecl>
     bindingDecl;
 
@@ -368,33 +581,41 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, BindingDecl>
 /// \code
 ///   extern "C" {}
 /// \endcode
-/// linkageSpecDecl()
-///   matches "extern "C" {}"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{linkageSpecDecl()}
+/// matches \match{extern "C" {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, LinkageSpecDecl>
     linkageSpecDecl;
 
 /// Matches a declaration of anything that could have a name.
 ///
 /// Example matches \c X, \c S, the anonymous union type, \c i, and \c U;
+/// Given
 /// \code
 ///   typedef int X;
-///   struct S {
-///     union {
-///       int i;
-///     } U;
-///   };
-/// \endcode
+///   struct S { union { int i; } U; };
+/// \endcode
+/// The matcher \matcher{namedDecl()}
+/// matches \match{typedef int X},
+/// \match{std=c$struct S { union { int i; } U; }}, \match{int i},
+/// the unnamed union\match{type=name$} and the variable
+/// \match{union { int i; } U},
+/// with \match{type=name;count=2;std=c++$S} matching twice in C++.
+/// Once for the implicit class declaration and once for the declaration itself.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, NamedDecl> namedDecl;
 
 /// Matches a declaration of label.
 ///
 /// Given
 /// \code
-///   goto FOO;
-///   FOO: bar();
+///   void bar();
+///   void foo() {
+///     goto FOO;
+///     FOO: bar();
+///   }
 /// \endcode
-/// labelDecl()
-///   matches 'FOO:'
+/// The matcher \matcher{type=none$labelDecl()}
+/// matches \match{FOO: bar()}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, LabelDecl> labelDecl;
 
 /// Matches a declaration of a namespace.
@@ -404,8 +625,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, LabelDecl> labelDecl;
 ///   namespace {}
 ///   namespace test {}
 /// \endcode
-/// namespaceDecl()
-///   matches "namespace {}" and "namespace test {}"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{namespaceDecl()}
+/// matches \match{namespace {}} and \match{namespace test {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, NamespaceDecl>
     namespaceDecl;
 
@@ -416,38 +638,53 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, NamespaceDecl>
 ///   namespace test {}
 ///   namespace alias = ::test;
 /// \endcode
-/// namespaceAliasDecl()
-///   matches "namespace alias" but not "namespace test"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{namespaceAliasDecl()}
+/// matches \match{namespace alias = ::test},
+/// but does not match \nomatch{namespace test {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, NamespaceAliasDecl>
     namespaceAliasDecl;
 
 /// Matches class, struct, and union declarations.
 ///
-/// Example matches \c X, \c Z, \c U, and \c S
+/// Given
 /// \code
 ///   class X;
 ///   template<class T> class Z {};
 ///   struct S {};
 ///   union U {};
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{recordDecl()}
+/// matches \match{class X} once, and the rest of the declared records twice,
+/// once for their written definition and once for their implicit declaration:
+/// \match{type=name;count=2$Z}, \match{type=name;count=2$S} and
+/// \match{type=name;count=2$U}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, RecordDecl> recordDecl;
 
 /// Matches C++ class declarations.
 ///
-/// Example matches \c X, \c Z
+/// Given
 /// \code
 ///   class X;
 ///   template<class T> class Z {};
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxRecordDecl()}
+/// matches \match{class X} once, and \match{type=name;count=2$Z} twice,
+/// once for the written definition and once for the implicit declaration.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXRecordDecl>
     cxxRecordDecl;
 
 /// Matches C++ class template declarations.
 ///
-/// Example matches \c Z
+/// Given
 /// \code
 ///   template<class T> class Z {};
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplateDecl()}
+/// matches \match{template<class T> class Z {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ClassTemplateDecl>
     classTemplateDecl;
 
@@ -459,8 +696,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ClassTemplateDecl>
 ///   template<> class A<double> {};
 ///   A<int> a;
 /// \endcode
-/// classTemplateSpecializationDecl()
-///   matches the specializations \c A<int> and \c A<double>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplateSpecializationDecl()}
+/// matches \match{type=typestr$class A<int>}
+/// and \match{type=typestr$class A<double>}.
 extern const internal::VariadicDynCastAllOfMatcher<
     Decl, ClassTemplateSpecializationDecl>
     classTemplateSpecializationDecl;
@@ -472,14 +711,15 @@ extern const internal::VariadicDynCastAllOfMatcher<
 ///   template<class T1, class T2, int I>
 ///   class A {};
 ///
-///   template<class T, int I>
-///   class A<T, T*, I> {};
+///   template<class T, int I> class A<T, T*, I> {};
 ///
 ///   template<>
 ///   class A<int, int, 1> {};
 /// \endcode
-/// classTemplatePartialSpecializationDecl()
-///   matches the specialization \c A<T,T*,I> but not \c A<int,int,1>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplatePartialSpecializationDecl()}
+/// matches \match{template<class T, int I> class A<T, T*, I> {}},
+/// but does not match \nomatch{A<int, int, 1>}.
 extern const internal::VariadicDynCastAllOfMatcher<
     Decl, ClassTemplatePartialSpecializationDecl>
     classTemplatePartialSpecializationDecl;
@@ -491,8 +731,9 @@ extern const internal::VariadicDynCastAllOfMatcher<
 /// \code
 ///   class X { int y; };
 /// \endcode
-/// declaratorDecl()
-///   matches \c int y.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{declaratorDecl()}
+/// matches \match{int y}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, DeclaratorDecl>
     declaratorDecl;
 
@@ -502,8 +743,8 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, DeclaratorDecl>
 /// \code
 ///   void f(int x);
 /// \endcode
-/// parmVarDecl()
-///   matches \c int x.
+/// The matcher \matcher{parmVarDecl()}
+/// matches \match{int x}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ParmVarDecl>
     parmVarDecl;
 
@@ -516,29 +757,36 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ParmVarDecl>
 ///     int a;
 ///   };
 /// \endcode
-/// accessSpecDecl()
-///   matches 'public:'
+/// \compile_args{-std=c++}
+/// The matcher \matcher{accessSpecDecl()}
+/// matches \match{public:}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, AccessSpecDecl>
     accessSpecDecl;
 
 /// Matches class bases.
 ///
-/// Examples matches \c public virtual B.
+/// Given
 /// \code
 ///   class B {};
-///   class C : public virtual B {};
+///   class C : public B {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasDirectBase(cxxBaseSpecifier()))}
+/// matches \match{class C : public B {}}.
 extern const internal::VariadicAllOfMatcher<CXXBaseSpecifier> cxxBaseSpecifier;
 
 /// Matches constructor initializers.
 ///
-/// Examples matches \c i(42).
+/// Given
 /// \code
 ///   class C {
 ///     C() : i(42) {}
 ///     int i;
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxCtorInitializer()}
+/// matches \match{i(42)}.
 extern const internal::VariadicAllOfMatcher<CXXCtorInitializer>
     cxxCtorInitializer;
 
@@ -549,8 +797,10 @@ extern const internal::VariadicAllOfMatcher<CXXCtorInitializer>
 ///   template <typename T> struct C {};
 ///   C<int> c;
 /// \endcode
-/// templateArgument()
-///   matches 'int' in C<int>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{templateSpecializationType(hasAnyTemplateArgument(templateArgument()))}
+/// matches \match{type=typestr$C<int>}.
 extern const internal::VariadicAllOfMatcher<TemplateArgument> templateArgument;
 
 /// Matches template arguments (with location info).
@@ -560,8 +810,9 @@ extern const internal::VariadicAllOfMatcher<TemplateArgument> templateArgument;
 ///   template <typename T> struct C {};
 ///   C<int> c;
 /// \endcode
-/// templateArgumentLoc()
-///   matches 'int' in C<int>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{templateArgumentLoc()}
+/// matches \match{int} in C<int>.
 extern const internal::VariadicAllOfMatcher<TemplateArgumentLoc>
     templateArgumentLoc;
 
@@ -569,11 +820,15 @@ extern const internal::VariadicAllOfMatcher<TemplateArgumentLoc>
 ///
 /// Given
 /// \code
-///   template <typename T> class X { };
-///   X<int> xi;
+///   template<template <typename> class S> class X {};
+///   template<typename T> class Y {};
+///   X<Y> xi;
 /// \endcode
-/// templateName()
-///   matches 'X' in X<int>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasAnyTemplateArgument(
+///               refersToTemplate(templateName())))}
+/// matches the specialization \match{type=typestr$class X<Y>}
 extern const internal::VariadicAllOfMatcher<TemplateName> templateName;
 
 /// Matches non-type template parameter declarations.
@@ -582,8 +837,10 @@ extern const internal::VariadicAllOfMatcher<TemplateName> templateName;
 /// \code
 ///   template <typename T, int N> struct C {};
 /// \endcode
-/// nonTypeTemplateParmDecl()
-///   matches 'N', but not 'T'.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{nonTypeTemplateParmDecl()}
+/// matches \match{int N},
+/// but does not match \nomatch{typename T}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl,
                                                    NonTypeTemplateParmDecl>
     nonTypeTemplateParmDecl;
@@ -594,8 +851,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl,
 /// \code
 ///   template <typename T, int N> struct C {};
 /// \endcode
-/// templateTypeParmDecl()
-///   matches 'T', but not 'N'.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{templateTypeParmDecl()}
+/// matches \match{typename T},
+/// but does not \nomatch{int N}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TemplateTypeParmDecl>
     templateTypeParmDecl;
 
@@ -605,8 +864,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, TemplateTypeParmDecl>
 /// \code
 ///   template <template <typename> class Z, int N> struct C {};
 /// \endcode
-/// templateTypeParmDecl()
-///   matches 'Z', but not 'N'.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{templateTemplateParmDecl()}
+/// matches \match{template <typename> class Z},
+/// but does not match \nomatch{int N}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl,
                                                    TemplateTemplateParmDecl>
     templateTemplateParmDecl;
@@ -614,20 +875,31 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl,
 /// Matches public C++ declarations and C++ base specifers that specify public
 /// inheritance.
 ///
-/// Examples:
+/// Given
 /// \code
 ///   class C {
-///   public:    int a; // fieldDecl(isPublic()) matches 'a'
+///   public:    int a;
 ///   protected: int b;
 ///   private:   int c;
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl(isPublic())}
+/// matches \match{int a}.
 ///
+/// Given
 /// \code
 ///   class Base {};
-///   class Derived1 : public Base {}; // matches 'Base'
-///   struct Derived2 : Base {}; // matches 'Base'
+///   class Derived1 : public Base {};
+///   struct Derived2 : Base {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPublic()).bind("base")))}
+/// matches \match{class Derived1 : public Base {}} and
+/// \match{struct Derived2 : Base {}},
+/// with \matcher{type=sub$cxxBaseSpecifier(isPublic())} matching
+/// \match{sub=base$public Base} and \match{sub=base$Base}.
 AST_POLYMORPHIC_MATCHER(isPublic,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(Decl,
                                                         CXXBaseSpecifier)) {
@@ -637,19 +909,28 @@ AST_POLYMORPHIC_MATCHER(isPublic,
 /// Matches protected C++ declarations and C++ base specifers that specify
 /// protected inheritance.
 ///
-/// Examples:
+/// Given
 /// \code
 ///   class C {
 ///   public:    int a;
-///   protected: int b; // fieldDecl(isProtected()) matches 'b'
+///   protected: int b;
 ///   private:   int c;
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl(isProtected())}
+/// matches \match{int b}.
 ///
 /// \code
 ///   class Base {};
-///   class Derived : protected Base {}; // matches 'Base'
+///   class Derived : protected Base {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isProtected()).bind("base")))}
+/// matches \match{class Derived : protected Base {}}, with
+/// \matcher{type=sub$cxxBaseSpecifier(isProtected())} matching
+/// \match{sub=base$Base}.
 AST_POLYMORPHIC_MATCHER(isProtected,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(Decl,
                                                         CXXBaseSpecifier)) {
@@ -659,20 +940,30 @@ AST_POLYMORPHIC_MATCHER(isProtected,
 /// Matches private C++ declarations and C++ base specifers that specify private
 /// inheritance.
 ///
-/// Examples:
+/// Given
 /// \code
 ///   class C {
 ///   public:    int a;
 ///   protected: int b;
-///   private:   int c; // fieldDecl(isPrivate()) matches 'c'
+///   private:   int c;
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl(isPrivate())}
+/// matches \match{int c}.
 ///
 /// \code
 ///   struct Base {};
-///   struct Derived1 : private Base {}; // matches 'Base'
-///   class Derived2 : Base {}; // matches 'Base'
+///   struct Derived1 : private Base {}; // \match{Base}
+///   class Derived2 : Base {}; // \match{Base}
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(isPrivate()).bind("base")))}
+/// matches \match{struct Derived1 : private Base {}} and
+/// \match{class Derived2 : Base {}}, with
+/// \matcher{type=sub$cxxBaseSpecifier(isPrivate())} matching
+/// \match{sub=base;count=2$Base} each time.
 AST_POLYMORPHIC_MATCHER(isPrivate,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(Decl,
                                                         CXXBaseSpecifier)) {
@@ -688,11 +979,11 @@ AST_POLYMORPHIC_MATCHER(isPrivate,
 ///     int b;
 ///   };
 /// \endcode
-/// fieldDecl(isBitField())
-///   matches 'int a;' but not 'int b;'.
-AST_MATCHER(FieldDecl, isBitField) {
-  return Node.isBitField();
-}
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl(isBitField())}
+/// matches \match{int a : 2},
+/// but does not match \nomatch{int b}.
+AST_MATCHER(FieldDecl, isBitField) { return Node.isBitField(); }
 
 /// Matches non-static data members that are bit-fields of the specified
 /// bit width.
@@ -705,8 +996,10 @@ AST_MATCHER(FieldDecl, isBitField) {
 ///     int c : 2;
 ///   };
 /// \endcode
-/// fieldDecl(hasBitWidth(2))
-///   matches 'int a;' and 'int c;' but not 'int b;'.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl(hasBitWidth(2))}
+/// matches \match{int a : 2} and \match{int c : 2},
+/// but not \nomatch{int b : 4}.
 AST_MATCHER_P(FieldDecl, hasBitWidth, unsigned, Width) {
   return Node.isBitField() &&
          Node.getBitWidthValue(Finder->getASTContext()) == Width;
@@ -722,10 +1015,14 @@ AST_MATCHER_P(FieldDecl, hasBitWidth, unsigned, Width) {
 ///     int c;
 ///   };
 /// \endcode
-/// fieldDecl(hasInClassInitializer(integerLiteral(equals(2))))
-///   matches 'int a;' but not 'int b;'.
-/// fieldDecl(hasInClassInitializer(anything()))
-///   matches 'int a;' and 'int b;' but not 'int c;'.
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{fieldDecl(hasInClassInitializer(integerLiteral(equals(2))))}
+/// matches \match{int a = 2},
+/// but does not match \nomatch{int b = 3}.
+/// The matcher \matcher{fieldDecl(hasInClassInitializer(anything()))}
+/// matches \match{int a = 2} and \match{int b = 3},
+/// but does not match \nomatch{int c}.
 AST_MATCHER_P(FieldDecl, hasInClassInitializer, internal::Matcher<Expr>,
               InnerMatcher) {
   const Expr *Initializer = Node.getInClassInitializer();
@@ -735,20 +1032,29 @@ AST_MATCHER_P(FieldDecl, hasInClassInitializer, internal::Matcher<Expr>,
 
 /// Determines whether the function is "main", which is the entry point
 /// into an executable program.
-AST_MATCHER(FunctionDecl, isMain) {
-  return Node.isMain();
-}
+///
+/// Given
+/// \code
+///   void f();
+///   int main() {}
+/// \endcode
+///
+/// The matcher \matcher{functionDecl(isMain())} matches \match{int main() {}}.
+AST_MATCHER(FunctionDecl, isMain) { return Node.isMain(); }
 
 /// Matches the specialized template of a specialization declaration.
 ///
 /// Given
 /// \code
-///   template<typename T> class A {}; #1
-///   template<> class A<int> {}; #2
+///   template<typename T> class A {}; // #1
+///   template<> class A<int> {}; // #2
 /// \endcode
-/// classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl()))
-///   matches '#2' with classTemplateDecl() matching the class template
-///   declaration of 'A' at #1.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasSpecializedTemplate(classTemplateDecl().bind("ctd")))}
+/// matches \match{template<> class A<int> {}},
+/// with \matcher{type=sub$classTemplateDecl()} matching the class template
+/// declaration \match{sub=ctd$template <typename T> class A {}}.
 AST_MATCHER_P(ClassTemplateSpecializationDecl, hasSpecializedTemplate,
               internal::Matcher<ClassTemplateDecl>, InnerMatcher) {
   const ClassTemplateDecl* Decl = Node.getSpecializedTemplate();
@@ -758,6 +1064,22 @@ AST_MATCHER_P(ClassTemplateSpecializationDecl, hasSpecializedTemplate,
 
 /// Matches an entity that has been implicitly added by the compiler (e.g.
 /// implicit default/copy constructors).
+///
+/// Given
+/// \code
+///   struct S {};
+///   void f(S obj) {
+///     S copy = obj;
+///     [&](){ return copy; };
+///   }
+/// \endcode
+/// \compile_args{-std=c++11}
+///
+/// The matcher \matcher{cxxConstructorDecl(isImplicit(), isCopyConstructor())}
+/// matches the implicit copy constructor of \match{S}.
+/// The matcher \matcher{lambdaExpr(forEachLambdaCapture(
+///     lambdaCapture(isImplicit())))} matches \match{[&](){ return copy; }},
+/// because it implicitly captures \c copy .
 AST_POLYMORPHIC_MATCHER(isImplicit,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Attr,
                                                         LambdaCapture)) {
@@ -774,17 +1096,21 @@ AST_POLYMORPHIC_MATCHER(isImplicit,
 ///   template<> class A<double> {};
 ///   A<int> a;
 ///
-///   template<typename T> f() {};
+///   template<typename T> void f() {};
 ///   void func() { f<int>(); };
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
 ///
-/// \endcode
-/// classTemplateSpecializationDecl(hasAnyTemplateArgument(
-///     refersToType(asString("int"))))
-///   matches the specialization \c A<int>
+/// The matcher \matcher{classTemplateSpecializationDecl(
+///                         hasAnyTemplateArgument(
+///                           refersToType(asString("int"))))}
+/// matches \match{type=typestr$class A<int>}.
 ///
-/// functionDecl(hasAnyTemplateArgument(refersToType(asString("int"))))
-///   matches the specialization \c f<int>
+/// The matcher
+/// \matcher{functionDecl(hasAnyTemplateArgument(
+///               refersToType(asString("int"))))}
+/// matches the instantiation of
+/// \match{void f() {}}.
 AST_POLYMORPHIC_MATCHER_P(
     hasAnyTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -807,12 +1133,10 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   }
 /// \endcode
 /// The matcher
-/// \code
-///   traverse(TK_IgnoreUnlessSpelledInSource,
+/// \matcher{traverse(TK_IgnoreUnlessSpelledInSource,
 ///     varDecl(hasInitializer(floatLiteral().bind("init")))
-///   )
-/// \endcode
-/// matches the variable declaration with "init" bound to the "3.0".
+///   )}
+///   matches \match{int i = 3.0} with "init" bound to \match{sub=init$3.0}.
 template <typename T>
 internal::Matcher<T> traverse(TraversalKind TK,
                               const internal::Matcher<T> &InnerMatcher) {
@@ -873,25 +1197,37 @@ traverse(TraversalKind TK, const internal::MapAnyOfHelper<T...> &InnerMatcher) {
 /// nodes are stripped off.
 ///
 /// Parentheses and explicit casts are not discarded.
+///
 /// Given
 /// \code
-///   class C {};
-///   C a = C();
-///   C b;
-///   C c = b;
-/// \endcode
-/// The matchers
-/// \code
-///    varDecl(hasInitializer(ignoringImplicit(cxxConstructExpr())))
-/// \endcode
-/// would match the declarations for a, b, and c.
-/// While
-/// \code
-///    varDecl(hasInitializer(cxxConstructExpr()))
+///   void f(int param) {
+///     int a = 0;
+///     int b = param;
+///     const int c = 0;
+///     const int d = param;
+///     int e = (0U);
+///     int f = (int)0.0;
+///     const int g = ((int)(((0))));
+///   }
 /// \endcode
-/// only match the declarations for b and c.
-AST_MATCHER_P(Expr, ignoringImplicit, internal::Matcher<Expr>,
-              InnerMatcher) {
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringImplicit(integerLiteral())))}
+/// matches \match{int a = 0} and \match{const int c = 0},
+/// but not \nomatch{int e = (0U)} and \nomatch{((int)(((0)))}.
+/// The matcher
+/// \matcher{varDecl(hasInitializer(integerLiteral()))}
+/// matches \match{int a = 0} and \match{const int c = 0},
+/// but not \nomatch{int e = (0U)} and \nomatch{((int)(((0)))}.
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringImplicit(declRefExpr())))}
+/// matches \match{int b = param} and \match{const int d = param}.
+/// The matcher
+/// \matcher{varDecl(hasInitializer(declRefExpr()))}
+/// matches neither \nomatch{int b = param} nor \nomatch{const int d = param},
+/// because an l-to-r-value cast happens.
+AST_MATCHER_P(Expr, ignoringImplicit, internal::Matcher<Expr>, InnerMatcher) {
   return InnerMatcher.matches(*Node.IgnoreImplicit(), Finder, Builder);
 }
 
@@ -902,24 +1238,32 @@ AST_MATCHER_P(Expr, ignoringImplicit, internal::Matcher<Expr>,
 /// Given
 /// \code
 ///   int arr[5];
-///   int a = 0;
+///   const int a = 0;
 ///   char b = 0;
 ///   const int c = a;
 ///   int *d = arr;
 ///   long e = (long) 0l;
 /// \endcode
-/// The matchers
-/// \code
-///    varDecl(hasInitializer(ignoringImpCasts(integerLiteral())))
-///    varDecl(hasInitializer(ignoringImpCasts(declRefExpr())))
-/// \endcode
-/// would match the declarations for a, b, c, and d, but not e.
-/// While
-/// \code
-///    varDecl(hasInitializer(integerLiteral()))
-///    varDecl(hasInitializer(declRefExpr()))
-/// \endcode
-/// only match the declarations for a.
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringImpCasts(integerLiteral())))}
+/// matches \match{const int a = 0} and \match{char b = 0},
+/// but does not match \nomatch{long e = (long) 0l} because of the c-style
+/// case.
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringImpCasts(declRefExpr())))}
+/// matches \match{const int c = a} and \match{int *d = arr}.
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(integerLiteral()))}
+/// matches \match{const int a = 0},
+/// but does not match \nomatch{char b = 0} because of the implicit cast to
+/// \c char or \nomatch{long e = (long) 0l} because of the c-style cast.
+///
+/// The matcher \matcher{varDecl(hasInitializer(declRefExpr()))}
+/// does not match \nomatch{const int c = a} because \c a is cast from an
+/// l- to an r-value or \nomatch{int *d = arr} because the array-to-pointer
+/// decay.
 AST_MATCHER_P(Expr, ignoringImpCasts,
               internal::Matcher<Expr>, InnerMatcher) {
   return InnerMatcher.matches(*Node.IgnoreImpCasts(), Finder, Builder);
@@ -936,12 +1280,15 @@ AST_MATCHER_P(Expr, ignoringImpCasts,
 ///   void* c = reinterpret_cast<char*>(0);
 ///   char d = char(0);
 /// \endcode
+/// \compile_args{-std=c++}
 /// The matcher
-///    varDecl(hasInitializer(ignoringParenCasts(integerLiteral())))
-/// would match the declarations for a, b, c, and d.
-/// while
-///    varDecl(hasInitializer(integerLiteral()))
-/// only match the declaration for a.
+/// \matcher{varDecl(hasInitializer(ignoringParenCasts(integerLiteral())))}
+/// matches \match{int a = 0}, \match{char b = (0)},
+/// \match{void* c = reinterpret_cast<char*>(0)} and \match{type=name$d}.
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(integerLiteral()))}
+/// matches \match{int a = 0}.
 AST_MATCHER_P(Expr, ignoringParenCasts, internal::Matcher<Expr>, InnerMatcher) {
   return InnerMatcher.matches(*Node.IgnoreParenCasts(), Finder, Builder);
 }
@@ -959,14 +1306,25 @@ AST_MATCHER_P(Expr, ignoringParenCasts, internal::Matcher<Expr>, InnerMatcher) {
 ///   int *d = (arr);
 ///   long e = ((long) 0l);
 /// \endcode
-/// The matchers
-///    varDecl(hasInitializer(ignoringParenImpCasts(integerLiteral())))
-///    varDecl(hasInitializer(ignoringParenImpCasts(declRefExpr())))
-/// would match the declarations for a, b, c, and d, but not e.
-/// while
-///    varDecl(hasInitializer(integerLiteral()))
-///    varDecl(hasInitializer(declRefExpr()))
-/// would only match the declaration for a.
+/// \compile_args{-std=c++}
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringParenImpCasts(integerLiteral())))}
+/// matches \match{int a = 0} and \match{char b = (0)},
+/// but does not match \nomatch{long e = ((long) 0l)} because of the c-style
+/// cast.
+///
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringParenImpCasts(declRefExpr())))}
+/// matches \match{const int c = a} and \match{int *d = (arr)}.
+///
+/// The matcher \matcher{varDecl(hasInitializer(integerLiteral()))} matches
+/// \match{int a = 0}, but does not match \nomatch{char b = (0)} or
+/// \nomatch{long e = ((long) 0l)} because of the casts.
+///
+/// The matcher \matcher{varDecl(hasInitializer(declRefExpr()))}
+/// does not match \nomatch{const int c = a} because of the l- to r-value cast,
+/// or \nomatch{int *d = (arr)} because of the array-to-pointer decay.
 AST_MATCHER_P(Expr, ignoringParenImpCasts,
               internal::Matcher<Expr>, InnerMatcher) {
   return InnerMatcher.matches(*Node.IgnoreParenImpCasts(), Finder, Builder);
@@ -979,10 +1337,8 @@ AST_MATCHER_P(Expr, ignoringParenImpCasts,
 ///   void (*fp)(void);
 /// \endcode
 /// The matcher
-/// \code
-///   varDecl(hasType(pointerType(pointee(ignoringParens(functionType())))))
-/// \endcode
-/// would match the declaration for fp.
+/// \matcher{varDecl(hasType(pointerType(pointee(ignoringParens(functionType())))))}
+/// matches \match{void (*fp)(void)}.
 AST_MATCHER_P_OVERLOAD(QualType, ignoringParens, internal::Matcher<QualType>,
                        InnerMatcher, 0) {
   return InnerMatcher.matches(Node.IgnoreParens(), Finder, Builder);
@@ -995,10 +1351,9 @@ AST_MATCHER_P_OVERLOAD(QualType, ignoringParens, internal::Matcher<QualType>,
 ///   const char* str = ("my-string");
 /// \endcode
 /// The matcher
-/// \code
-///   implicitCastExpr(hasSourceExpression(ignoringParens(stringLiteral())))
-/// \endcode
-/// would match the implicit cast resulting from the assignment.
+/// \matcher{implicitCastExpr(hasSourceExpression(ignoringParens(stringLiteral())))}
+/// would match the implicit cast resulting from the assignment
+/// \match{("my-string")}.
 AST_MATCHER_P_OVERLOAD(Expr, ignoringParens, internal::Matcher<Expr>,
                        InnerMatcher, 1) {
   const Expr *E = Node.IgnoreParens();
@@ -1015,9 +1370,13 @@ AST_MATCHER_P_OVERLOAD(Expr, ignoringParens, internal::Matcher<Expr>,
 /// sizeof is known.
 /// \code
 ///   template<typename T>
-///   void f(T x, T y) { sizeof(sizeof(T() + T()); }
+///   void f(T x, T y) { sizeof(T() + T()); }
 /// \endcode
-/// expr(isInstantiationDependent()) matches sizeof(sizeof(T() + T())
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{expr(isInstantiationDependent())}
+/// matches \match{sizeof(T() + T())},
+/// \match{(T() + T())},
+/// \match{T() + T()} and two time \match{count=2$T()}.
 AST_MATCHER(Expr, isInstantiationDependent) {
   return Node.isInstantiationDependent();
 }
@@ -1033,7 +1392,9 @@ AST_MATCHER(Expr, isInstantiationDependent) {
 ///     x + y;
 ///   }
 /// \endcode
-/// expr(isTypeDependent()) matches x + y
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{expr(isTypeDependent())}
+/// matches \match{x + y} and \match{x}.
 AST_MATCHER(Expr, isTypeDependent) { return Node.isTypeDependent(); }
 
 /// Matches expression that are value-dependent because they contain a
@@ -1044,7 +1405,9 @@ AST_MATCHER(Expr, isTypeDependent) { return Node.isTypeDependent(); }
 /// \code
 ///   template<int Size> int f() { return Size; }
 /// \endcode
-/// expr(isValueDependent()) matches return Size
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{expr(isValueDependent())}
+/// matches the return value \match{Size}.
 AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 
 /// Matches templateSpecializationType, class template specializations,
@@ -1054,18 +1417,21 @@ AST_MATCHER(Expr, isValueDependent) { return Node.isValueDependent(); }
 /// Given
 /// \code
 ///   template<typename T, typename U> class A {};
-///   A<bool, int> b;
-///   A<int, bool> c;
+///   A<double, int> b;
+///   A<int, double> c;
 ///
 ///   template<typename T> void f() {}
 ///   void func() { f<int>(); };
 /// \endcode
-/// classTemplateSpecializationDecl(hasTemplateArgument(
-///     1, refersToType(asString("int"))))
-///   matches the specialization \c A<bool, int>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasTemplateArgument(
+///     1, refersToType(asString("int"))))}
+/// matches the specialization \match{type=typestr$class A<double, int>}.
 ///
-/// functionDecl(hasTemplateArgument(0, refersToType(asString("int"))))
-///   matches the specialization \c f<int>
+/// The matcher \matcher{functionDecl(hasTemplateArgument(0,
+///                         refersToType(asString("int"))))}
+/// matches the specialization of \match{void f() {}}.
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -1086,8 +1452,10 @@ AST_POLYMORPHIC_MATCHER_P2(
 ///   template<typename T> struct C {};
 ///   C<int> c;
 /// \endcode
-/// classTemplateSpecializationDecl(templateArgumentCountIs(1))
-///   matches C<int>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(templateArgumentCountIs(1))}
+/// matches \match{type=typestr$struct C<int>}.
 AST_POLYMORPHIC_MATCHER_P(
     templateArgumentCountIs,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -1104,9 +1472,11 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   template<typename T> struct A {};
 ///   A<X> a;
 /// \endcode
-/// classTemplateSpecializationDecl(hasAnyTemplateArgument(refersToType(
-///   recordType(hasDeclaration(recordDecl(hasName("X")))))))
-/// matches the specialization of \c struct A generated by \c A<X>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasAnyTemplateArgument(refersToType(
+///             recordType(hasDeclaration(recordDecl(hasName("X")))))))}
+/// matches the specialization \match{type=typestr$struct A<struct X>}.
 AST_MATCHER_P(TemplateArgument, refersToType,
               internal::Matcher<QualType>, InnerMatcher) {
   if (Node.getKind() != TemplateArgument::Type)
@@ -1122,9 +1492,11 @@ AST_MATCHER_P(TemplateArgument, refersToType,
 ///   template<typename T> class Y {};
 ///   X<Y> xi;
 /// \endcode
-/// classTemplateSpecializationDecl(hasAnyTemplateArgument(
-///     refersToTemplate(templateName())))
-///   matches the specialization \c X<Y>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasAnyTemplateArgument(
+///               refersToTemplate(templateName())))}
+/// matches the specialization \match{type=typestr$class X<Y>}
 AST_MATCHER_P(TemplateArgument, refersToTemplate,
               internal::Matcher<TemplateName>, InnerMatcher) {
   if (Node.getKind() != TemplateArgument::Template)
@@ -1141,10 +1513,13 @@ AST_MATCHER_P(TemplateArgument, refersToTemplate,
 ///   template<int(B::*next_ptr)> struct A {};
 ///   A<&B::next> a;
 /// \endcode
-/// classTemplateSpecializationDecl(hasAnyTemplateArgument(
-///     refersToDeclaration(fieldDecl(hasName("next")))))
-///   matches the specialization \c A<&B::next> with \c fieldDecl(...) matching
-///     \c B::next
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{classTemplateSpecializationDecl(hasAnyTemplateArgument(
+///     refersToDeclaration(fieldDecl(hasName("next")).bind("next"))))}
+/// matches the specialization \match{type=typestr$struct A<&B::next>}
+/// with \matcher{type=sub$fieldDecl(hasName("next"))} matching
+/// \match{sub=next$int next}.
 AST_MATCHER_P(TemplateArgument, refersToDeclaration,
               internal::Matcher<Decl>, InnerMatcher) {
   if (Node.getKind() == TemplateArgument::Declaration)
@@ -1160,10 +1535,13 @@ AST_MATCHER_P(TemplateArgument, refersToDeclaration,
 ///   template<int(B::*next_ptr)> struct A {};
 ///   A<&B::next> a;
 /// \endcode
-/// templateSpecializationType(hasAnyTemplateArgument(
-///   isExpr(hasDescendant(declRefExpr(to(fieldDecl(hasName("next"))))))))
-///   matches the specialization \c A<&B::next> with \c fieldDecl(...) matching
-///     \c B::next
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{templateSpecializationType(hasAnyTemplateArgument(
+///   isExpr(hasDescendant(declRefExpr(to(fieldDecl(hasName("next")).bind("next")))))))}
+/// matches the specialization \match{type=typestr$A<&struct B::next>}
+/// with \matcher{type=sub$fieldDecl(hasName("next"))} matching
+/// \match{sub=next$int next}.
 AST_MATCHER_P(TemplateArgument, isExpr, internal::Matcher<Expr>, InnerMatcher) {
   if (Node.getKind() == TemplateArgument::Expression)
     return InnerMatcher.matches(*Node.getAsExpr(), Finder, Builder);
@@ -1177,10 +1555,12 @@ AST_MATCHER_P(TemplateArgument, isExpr, internal::Matcher<Expr>, InnerMatcher) {
 ///   template<int T> struct C {};
 ///   C<42> c;
 /// \endcode
-/// classTemplateSpecializationDecl(
-///   hasAnyTemplateArgument(isIntegral()))
-///   matches the implicit instantiation of C in C<42>
-///   with isIntegral() matching 42.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplateSpecializationDecl(
+///   hasAnyTemplateArgument(isIntegral()))}
+/// matches the implicitly declared specialization
+/// \match{type=typestr$struct C<42>} from the instantiation for the type of the
+/// variable \c c .
 AST_MATCHER(TemplateArgument, isIntegral) {
   return Node.getKind() == TemplateArgument::Integral;
 }
@@ -1192,9 +1572,12 @@ AST_MATCHER(TemplateArgument, isIntegral) {
 ///   template<int T> struct C {};
 ///   C<42> c;
 /// \endcode
-/// classTemplateSpecializationDecl(
-///   hasAnyTemplateArgument(refersToIntegralType(asString("int"))))
-///   matches the implicit instantiation of C in C<42>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplateSpecializationDecl(
+///   hasAnyTemplateArgument(refersToIntegralType(asString("int"))))}
+/// matches the implicitly declared specialization
+/// \match{type=typestr$struct C<42>} from the instantiation for the type of the
+/// variable \c c .
 AST_MATCHER_P(TemplateArgument, refersToIntegralType,
               internal::Matcher<QualType>, InnerMatcher) {
   if (Node.getKind() != TemplateArgument::Integral)
@@ -1213,9 +1596,12 @@ AST_MATCHER_P(TemplateArgument, refersToIntegralType,
 ///   template<int T> struct C {};
 ///   C<42> c;
 /// \endcode
-/// classTemplateSpecializationDecl(
-///   hasAnyTemplateArgument(equalsIntegralValue("42")))
-///   matches the implicit instantiation of C in C<42>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{classTemplateSpecializationDecl(
+///   hasAnyTemplateArgument(equalsIntegralValue("42")))}
+/// matches the implicitly declared specialization
+/// \match{type=typestr$struct C<42>} from the instantiation for the type of the
+/// variable \c c .
 AST_MATCHER_P(TemplateArgument, equalsIntegralValue,
               std::string, Value) {
   if (Node.getKind() != TemplateArgument::Integral)
@@ -1231,23 +1617,28 @@ AST_MATCHER_P(TemplateArgument, equalsIntegralValue,
 ///     int x = 0;
 ///   }
 /// \endcode
-/// autoreleasePoolStmt(stmt()) matches the declaration of "x"
-/// inside the autorelease pool.
+/// \compile_args{-ObjC}
+/// The matcher \matcher{autoreleasePoolStmt(stmt())} matches the declaration of
+/// \match{int x = 0} inside the autorelease pool.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
        ObjCAutoreleasePoolStmt> autoreleasePoolStmt;
 
 /// Matches any value declaration.
 ///
-/// Example matches A, B, C and F
+/// Given
 /// \code
 ///   enum X { A, B, C };
 ///   void F();
+///   int V = 0;
 /// \endcode
+/// The matcher \matcher{valueDecl()}
+/// matches \match{A}, \match{B}, \match{C}, \match{void F()}
+/// and \match{int V = 0}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ValueDecl> valueDecl;
 
 /// Matches C++ constructor declarations.
 ///
-/// Example matches Foo::Foo() and Foo::Foo(int)
+/// Given
 /// \code
 ///   class Foo {
 ///    public:
@@ -1255,93 +1646,134 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ValueDecl> valueDecl;
 ///     Foo(int);
 ///     int DoSomething();
 ///   };
+///
+///   struct Bar {};
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxConstructorDecl()}
+/// matches \match{Foo()} and \match{Foo(int)}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXConstructorDecl>
     cxxConstructorDecl;
 
 /// Matches explicit C++ destructor declarations.
 ///
-/// Example matches Foo::~Foo()
+/// Given
 /// \code
 ///   class Foo {
 ///    public:
 ///     virtual ~Foo();
 ///   };
+///
+///   struct Bar {};
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxDestructorDecl()}
+/// matches \match{virtual ~Foo()}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXDestructorDecl>
     cxxDestructorDecl;
 
 /// Matches enum declarations.
 ///
-/// Example matches X
+/// Given
 /// \code
-///   enum X {
-///     A, B, C
-///   };
+///   enum X { A, B, C };
 /// \endcode
+///
+/// The matcher \matcher{enumDecl()}
+/// matches the enum \match{enum X { A, B, C }}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, EnumDecl> enumDecl;
 
 /// Matches enum constants.
 ///
-/// Example matches A, B, C
+/// Given
 /// \code
 ///   enum X {
 ///     A, B, C
 ///   };
 /// \endcode
+/// The matcher \matcher{enumConstantDecl()}
+/// matches \match{A}, \match{B} and \match{C}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, EnumConstantDecl>
     enumConstantDecl;
 
 /// Matches tag declarations.
 ///
-/// Example matches X, Z, U, S, E
+/// Given
 /// \code
 ///   class X;
 ///   template<class T> class Z {};
 ///   struct S {};
 ///   union U {};
-///   enum E {
-///     A, B, C
-///   };
+///   enum E { A, B, C };
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+///
+/// The matcher \matcher{tagDecl()}
+/// matches \match{class X}, \match{class Z {}}, the implicit class
+/// declaration \match{class Z}, \match{struct S {}},
+/// the implicit class declaration \match{struct S}, \match{union U {}},
+/// the implicit class declaration \match{union U}
+/// and \match{enum E { A, B, C }}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TagDecl> tagDecl;
 
 /// Matches method declarations.
 ///
-/// Example matches y
+/// Given
 /// \code
 ///   class X { void y(); };
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxMethodDecl()}
+/// matches \match{void y()}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXMethodDecl>
     cxxMethodDecl;
 
 /// Matches conversion operator declarations.
 ///
-/// Example matches the operator.
+/// Given
 /// \code
 ///   class X { operator int() const; };
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxConversionDecl()}
+/// matches \match{operator int() const}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXConversionDecl>
     cxxConversionDecl;
 
 /// Matches user-defined and implicitly generated deduction guide.
 ///
-/// Example matches the deduction guide.
+/// Given
 /// \code
 ///   template<typename T>
-///   class X { X(int) };
+///   class X { X(int); };
 ///   X(int) -> X<int>;
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxDeductionGuideDecl()}
+/// matches the written deduction guide
+/// \match{type=typestr$auto (int) -> X<int>},
+/// the implicit copy deduction guide \match{type=typestr$auto (int) -> X<T>}
+/// and the implicitly declared deduction guide
+/// \match{type=typestr$auto (X<T>) -> X<T>}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, CXXDeductionGuideDecl>
     cxxDeductionGuideDecl;
 
 /// Matches concept declarations.
 ///
-/// Example matches integral
+/// Given
 /// \code
-///   template<typename T>
-///   concept integral = std::is_integral_v<T>;
+///   template<typename T> concept my_concept = true;
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++20-or-later}
+///
+/// The matcher \matcher{conceptDecl()}
+/// matches \match{template<typename T>
+/// concept my_concept = true}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ConceptDecl>
     conceptDecl;
 
@@ -1353,17 +1785,29 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ConceptDecl>
 /// Example matches a
 /// \code
 ///   int a;
+///   struct Foo {
+///     int x;
+///   };
+///   void bar(int val);
 /// \endcode
+///
+/// The matcher \matcher{varDecl()}
+/// matches \match{int a} and \match{int val}, but not \nomatch{int x}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, VarDecl> varDecl;
 
 /// Matches field declarations.
 ///
 /// Given
 /// \code
-///   class X { int m; };
+///   int a;
+///   struct Foo {
+///     int x;
+///   };
+///   void bar(int val);
 /// \endcode
-/// fieldDecl()
-///   matches 'm'.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{fieldDecl()}
+/// matches \match{int x}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, FieldDecl> fieldDecl;
 
 /// Matches indirect field declarations.
@@ -1372,17 +1816,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, FieldDecl> fieldDecl;
 /// \code
 ///   struct X { struct { int a; }; };
 /// \endcode
-/// indirectFieldDecl()
-///   matches 'a'.
+/// The matcher \matcher{indirectFieldDecl()}
+/// matches \match{a}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, IndirectFieldDecl>
     indirectFieldDecl;
 
 /// Matches function declarations.
 ///
-/// Example matches f
+/// Given
 /// \code
 ///   void f();
 /// \endcode
+///
+/// The matcher \matcher{functionDecl()}
+/// matches \match{void f()}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, FunctionDecl>
     functionDecl;
 
@@ -1392,6 +1839,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, FunctionDecl>
 /// \code
 ///   template<class T> void f(T t) {}
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+///
+/// The matcher \matcher{functionTemplateDecl()}
+/// matches \match{template<class T> void f(T t) {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, FunctionTemplateDecl>
     functionTemplateDecl;
 
@@ -1401,28 +1852,32 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, FunctionTemplateDecl>
 /// \code
 ///   class X { friend void foo(); };
 /// \endcode
-/// friendDecl()
-///   matches 'friend void foo()'.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{friendDecl()}
+/// matches \match{friend void foo()}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, FriendDecl> friendDecl;
 
 /// Matches statements.
 ///
 /// Given
 /// \code
-///   { ++a; }
+///   void foo(int a) { { ++a; } }
 /// \endcode
-/// stmt()
-///   matches both the compound statement '{ ++a; }' and '++a'.
+/// The matcher \matcher{stmt()}
+/// matches the function body itself \match{{ { ++a; } }}, the compound
+/// statement \match{{ ++a; }}, the expression \match{++a} and \match{a}.
 extern const internal::VariadicAllOfMatcher<Stmt> stmt;
 
 /// Matches declaration statements.
 ///
 /// Given
 /// \code
-///   int a;
+///   void foo() {
+///     int a;
+///   }
 /// \endcode
-/// declStmt()
-///   matches 'int a'.
+/// The matcher \matcher{declStmt()}
+/// matches \match{int a;}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DeclStmt> declStmt;
 
 /// Matches member expressions.
@@ -1434,8 +1889,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, DeclStmt> declStmt;
 ///     int a; static int b;
 ///   };
 /// \endcode
-/// memberExpr()
-///   matches this->x, x, y.x, a, this->b
+/// \compile_args{-std=c++}
+/// The matcher \matcher{memberExpr()}
+/// matches \match{this->x}, \match{x}, \match{y.x}, \match{a}, \match{this->b}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, MemberExpr> memberExpr;
 
 /// Matches unresolved member expressions.
@@ -1448,8 +1904,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, MemberExpr> memberExpr;
 ///   };
 ///   template <class T> void h() { X x; x.f<T>(); x.g(); }
 /// \endcode
-/// unresolvedMemberExpr()
-///   matches x.f<T>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unresolvedMemberExpr()}
+/// matches \match{x.f<T>}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, UnresolvedMemberExpr>
     unresolvedMemberExpr;
 
@@ -1460,8 +1917,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, UnresolvedMemberExpr>
 /// \code
 ///   template <class T> void f() { T t; t.g(); }
 /// \endcode
-/// cxxDependentScopeMemberExpr()
-///   matches t.g
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxDependentScopeMemberExpr()}
+///   matches \match{t.g}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    CXXDependentScopeMemberExpr>
     cxxDependentScopeMemberExpr;
@@ -1470,15 +1928,22 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 ///
 /// Example matches x.y() and y()
 /// \code
-///   X x;
-///   x.y();
-///   y();
+///   struct X { void foo(); };
+///   void bar();
+///   void foobar() {
+///     X x;
+///     x.foo();
+///     bar();
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{callExpr()}
+/// matches \match{x.foo()} and \match{bar()};
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CallExpr> callExpr;
 
 /// Matches call expressions which were resolved using ADL.
 ///
-/// Example matches y(x) but not y(42) or NS::y(x).
+/// Given
 /// \code
 ///   namespace NS {
 ///     struct X {};
@@ -1494,25 +1959,46 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CallExpr> callExpr;
 ///     y(42); // Doesn't match
 ///     using NS::y;
 ///     y(x); // Found by both unqualified lookup and ADL, doesn't match
-//    }
+///    }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{callExpr(usesADL())}
+/// matches \match{y(x)}, but not \nomatch{y(42)} or \nomatch{NS::y(x)}.
 AST_MATCHER(CallExpr, usesADL) { return Node.usesADL(); }
 
 /// Matches lambda expressions.
 ///
-/// Example matches [&](){return 5;}
+/// Given
 /// \code
-///   [&](){return 5;}
+///   void f() {
+///     []() { return 5; };
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{lambdaExpr()} matches \match{[]() { return 5; }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, LambdaExpr> lambdaExpr;
 
 /// Matches member call expressions.
 ///
-/// Example matches x.y()
+/// Given
 /// \code
-///   X x;
-///   x.y();
+///   struct X {
+///     void y();
+///     void m() { y(); }
+///   };
+///   void f();
+///   void g() {
+///     X x;
+///     x.y();
+///     f();
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxMemberCallExpr()} matches \match{x.y()} and
+/// \match{y()}, but not \nomatch{f()}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXMemberCallExpr>
     cxxMemberCallExpr;
 
@@ -1525,6 +2011,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXMemberCallExpr>
 /// \code
 ///   [[NSString alloc] initWithString:@"Hello"]
 /// \endcode
+/// \compile_args{-ObjC}
+///
+/// The matcher \matcher{objcMessageExpr()} matches
+/// \match{[[NSString alloc] initWithString:@"Hello"]}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCMessageExpr>
     objcMessageExpr;
 
@@ -1534,6 +2024,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCMessageExpr>
 /// \code
 ///   NSString *s = @"abcd";
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCStringLiteral>
     objcStringLiteral;
 
@@ -1544,6 +2035,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCStringLiteral>
 ///   @interface Foo
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCInterfaceDecl>
     objcInterfaceDecl;
 
@@ -1554,6 +2046,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCInterfaceDecl>
 ///   @implementation Foo
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCImplementationDecl>
     objcImplementationDecl;
 
@@ -1564,6 +2057,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCImplementationDecl>
 ///   @protocol FooDelegate
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCProtocolDecl>
     objcProtocolDecl;
 
@@ -1574,6 +2068,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCProtocolDecl>
 ///   @interface Foo (Additions)
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCCategoryDecl>
     objcCategoryDecl;
 
@@ -1584,6 +2079,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCCategoryDecl>
 ///   @implementation Foo (Additions)
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCCategoryImplDecl>
     objcCategoryImplDecl;
 
@@ -1599,6 +2095,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCCategoryImplDecl>
 ///   - (void)method {}
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCMethodDecl>
     objcMethodDecl;
 
@@ -1612,6 +2109,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCMethodDecl>
 ///     printf("%d", p);
 ///   })
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, BlockDecl>
     blockDecl;
 
@@ -1624,6 +2122,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, BlockDecl>
 ///   }
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCIvarDecl>
     objcIvarDecl;
 
@@ -1635,6 +2134,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCIvarDecl>
 ///   @property BOOL enabled;
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCPropertyDecl>
     objcPropertyDecl;
 
@@ -1644,6 +2144,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, ObjCPropertyDecl>
 /// \code
 ///   @throw obj;
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtThrowStmt>
     objcThrowStmt;
 
@@ -1654,6 +2155,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtThrowStmt>
 ///   @try {}
 ///   @catch (...) {}
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtTryStmt>
     objcTryStmt;
 
@@ -1664,6 +2166,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtTryStmt>
 ///   @try {}
 ///   @catch (...) {}
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtCatchStmt>
     objcCatchStmt;
 
@@ -1674,6 +2177,7 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtCatchStmt>
 ///   @try {}
 ///   @finally {}
 /// \endcode
+/// \compile_args{-ObjC}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtFinallyStmt>
     objcFinallyStmt;
 
@@ -1682,8 +2186,21 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCAtFinallyStmt>
 ///
 /// Example matches std::string()
 /// \code
-///   const std::string str = std::string();
+///   struct A { ~A(); };
+///   void f(A);
+///   void g(A&);
+///   void h() {
+///     A a = A{};
+///     f(A{});
+///     f(a);
+///     g(a);
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{exprWithCleanups()} matches \match{A{}},
+/// \match{f(A{})} and \match{f(a)},
+/// but does not match passing \nomatch{g(a)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ExprWithCleanups>
     exprWithCleanups;
 
@@ -1693,17 +2210,29 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ExprWithCleanups>
 /// \code
 ///   int a[] = { 1, 2 };
 ///   struct B { int x, y; };
-///   B b = { 5, 6 };
+///   struct B b = { 5, 6 };
 /// \endcode
-/// initListExpr()
-///   matches "{ 1, 2 }" and "{ 5, 6 }"
+/// The matcher \matcher{initListExpr()}
+/// matches \match{{ 1, 2 }} and \match{{ 5, 6 }}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, InitListExpr>
     initListExpr;
 
 /// Matches the syntactic form of init list expressions
 /// (if expression have it).
-AST_MATCHER_P(InitListExpr, hasSyntacticForm,
-              internal::Matcher<Expr>, InnerMatcher) {
+///
+/// Given
+/// \code
+///   int a[] = { 1, 2 };
+///   struct B { int x, y; };
+///   struct B b = { 5, 6 };
+/// \endcode
+/// \compile_args{-std=c}
+///
+/// The matcher
+/// \matcher{initListExpr(hasSyntacticForm(expr().bind("syntactic")))}
+/// matches \match{{ 1, 2 }} and \match{{ 5, 6 }}.
+AST_MATCHER_P(InitListExpr, hasSyntacticForm, internal::Matcher<Expr>,
+              InnerMatcher) {
   const Expr *SyntForm = Node.getSyntacticForm();
   return (SyntForm != nullptr &&
           InnerMatcher.matches(*SyntForm, Finder, Builder));
@@ -1713,13 +2242,26 @@ AST_MATCHER_P(InitListExpr, hasSyntacticForm,
 ///
 /// Given
 /// \code
-///   std::vector<int> a({ 1, 2, 3 });
-///   std::vector<int> b = { 4, 5 };
+///   namespace std {
+///     template <typename T>
+///     class initializer_list {
+///       const T* begin;
+///       const T* end;
+///     };
+///   }
+///   template <typename T> class vector {
+///     public: vector(std::initializer_list<T>) {}
+///   };
+///
+///   vector<int> a({ 1, 2, 3 });
+///   vector<int> b = { 4, 5 };
 ///   int c[] = { 6, 7 };
-///   std::pair<int, int> d = { 8, 9 };
+///   struct pair { int x; int y; };
+///   pair d = { 8, 9 };
 /// \endcode
-/// cxxStdInitializerListExpr()
-///   matches "{ 1, 2, 3 }" and "{ 4, 5 }"
+/// \compile_args{-fno-delayed-template-parsing;-std=c++11-or-later,-nostdinc++}
+/// The matcher \matcher{cxxStdInitializerListExpr()}
+/// matches \match{{ 1, 2, 3 }} and \match{{ 4, 5 }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    CXXStdInitializerListExpr>
     cxxStdInitializerListExpr;
@@ -1728,10 +2270,12 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 ///
 /// Given
 /// \code
-///   point ptarray[10] = { [2].y = 1.0, [2].x = 2.0, [0].x = 1.0 };
+///   struct point { double x; double y; };
+///   struct point pt = { .x = 42.0 };
 /// \endcode
-/// implicitValueInitExpr()
-///   matches "[0].y" (implicitly)
+/// The matcher
+/// \matcher{initListExpr(has(implicitValueInitExpr().bind("implicit")))}
+/// matches \match{{ .x = 42.0 }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ImplicitValueInitExpr>
     implicitValueInitExpr;
 
@@ -1748,8 +2292,12 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ImplicitValueInitExpr>
 ///     }
 ///   };
 /// \endcode
-/// parenListExpr() matches "*this" but NOT matches (a, b) because (a, b)
-/// has a predefined type and is a ParenExpr, not a ParenListExpr.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{parenListExpr()}
+/// matches \match{(*this)},
+/// but does not match \nomatch{(a, b)}
+/// because (a, b) has a predefined type and is a ParenExpr, not a
+/// ParenListExpr.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ParenListExpr>
     parenListExpr;
 
@@ -1761,8 +2309,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ParenListExpr>
 ///   struct A { static const int n = N; };
 ///   struct B : public A<42> {};
 /// \endcode
-/// substNonTypeTemplateParmExpr()
-///   matches "N" in the right-hand side of "static const int n = N;"
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{substNonTypeTemplateParmExpr()}
+/// matches \match{N} in the right-hand side of "static const int n = N;"
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    SubstNonTypeTemplateParmExpr>
     substNonTypeTemplateParmExpr;
@@ -1774,19 +2323,21 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 ///   namespace X { int x; }
 ///   using X::x;
 /// \endcode
-/// usingDecl()
-///   matches \code using X::x \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{usingDecl()}
+///   matches \match{using X::x}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingDecl> usingDecl;
 
 /// Matches using-enum declarations.
 ///
 /// Given
 /// \code
-///   namespace X { enum x {...}; }
+///   namespace X { enum x { val1, val2 }; }
 ///   using enum X::x;
 /// \endcode
-/// usingEnumDecl()
-///   matches \code using enum X::x \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{usingEnumDecl()}
+///   matches \match{using enum X::x}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingEnumDecl>
     usingEnumDecl;
 
@@ -1797,8 +2348,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingEnumDecl>
 ///   namespace X { int x; }
 ///   using namespace X;
 /// \endcode
-/// usingDirectiveDecl()
-///   matches \code using namespace X \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{usingDirectiveDecl()}
+///   matches \match{using namespace X}
 extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingDirectiveDecl>
     usingDirectiveDecl;
 
@@ -1814,8 +2366,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingDirectiveDecl>
 ///     foo<T>();
 ///   }
 /// \endcode
-/// unresolvedLookupExpr()
-///   matches \code foo<T>() \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unresolvedLookupExpr()}
+/// matches \match{foo<T>}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, UnresolvedLookupExpr>
     unresolvedLookupExpr;
 
@@ -1828,8 +2381,9 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, UnresolvedLookupExpr>
 ///     using X::x;
 ///   };
 /// \endcode
-/// unresolvedUsingValueDecl()
-///   matches \code using X::x \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unresolvedUsingValueDecl()}
+///   matches \match{using X::x}
 extern const internal::VariadicDynCastAllOfMatcher<Decl,
                                                    UnresolvedUsingValueDecl>
     unresolvedUsingValueDecl;
@@ -1847,80 +2401,110 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl,
 ///     using typename Base<T>::Foo;
 ///   };
 /// \endcode
-/// unresolvedUsingTypenameDecl()
-///   matches \code using Base<T>::Foo \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unresolvedUsingTypenameDecl()}
+///   matches \match{using typename Base<T>::Foo}
 extern const internal::VariadicDynCastAllOfMatcher<Decl,
                                                    UnresolvedUsingTypenameDecl>
     unresolvedUsingTypenameDecl;
 
 /// Matches a constant expression wrapper.
 ///
-/// Example matches the constant in the case statement:
-///     (matcher = constantExpr())
+/// Given
 /// \code
-///   switch (a) {
-///   case 37: break;
+///   void f(int a) {
+///     switch (a) {
+///       case 37: break;
+///     }
 ///   }
 /// \endcode
+///
+/// The matcher \matcher{constantExpr()} matches \match{37}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ConstantExpr>
     constantExpr;
 
 /// Matches parentheses used in expressions.
 ///
-/// Example matches (foo() + 1)
+/// Given
 /// \code
 ///   int foo() { return 1; }
-///   int a = (foo() + 1);
+///   int bar() {
+///     int a = (foo() + 1);
+///   }
 /// \endcode
+///
+/// The matcher \matcher{parenExpr()} matches \match{(foo() + 1)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ParenExpr> parenExpr;
 
 /// Matches constructor call expressions (including implicit ones).
 ///
-/// Example matches string(ptr, n) and ptr within arguments of f
-///     (matcher = cxxConstructExpr())
+/// Given
 /// \code
+///   struct string {
+///     string(const char*);
+///     string(const char*s, int n);
+///   };
 ///   void f(const string &a, const string &b);
-///   char *ptr;
-///   int n;
-///   f(string(ptr, n), ptr);
+///   void foo(char *ptr, int n) {
+///     f(string(ptr, n), ptr);
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxConstructExpr()} matches \match{string(ptr, n)}
+/// and \match{ptr} within arguments of \c f .
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXConstructExpr>
     cxxConstructExpr;
 
 /// Matches unresolved constructor call expressions.
 ///
-/// Example matches T(t) in return statement of f
-///     (matcher = cxxUnresolvedConstructExpr())
+/// Given
 /// \code
 ///   template <typename T>
 ///   void f(const T& t) { return T(t); }
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+///
+/// The matcher \matcher{cxxUnresolvedConstructExpr()} matches
+/// \match{T(t)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    CXXUnresolvedConstructExpr>
     cxxUnresolvedConstructExpr;
 
 /// Matches implicit and explicit this expressions.
 ///
-/// Example matches the implicit this expression in "return i".
-///     (matcher = cxxThisExpr())
+/// Given
 /// \code
-/// struct foo {
-///   int i;
-///   int f() { return i; }
-/// };
+///   struct foo {
+///     int i;
+///     int f() { return i; }
+///     int g() { return this->i; }
+///   };
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxThisExpr()}
+/// matches \match{this} of \c this->i and the implicit \c this expression
+/// of \match{i}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXThisExpr>
     cxxThisExpr;
 
 /// Matches nodes where temporaries are created.
 ///
-/// Example matches FunctionTakesString(GetStringByValue())
-///     (matcher = cxxBindTemporaryExpr())
+/// Given
 /// \code
-///   FunctionTakesString(GetStringByValue());
-///   FunctionTakesStringByPointer(GetStringPointer());
-/// \endcode
-extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXBindTemporaryExpr>
+///   struct S {
+///     S() { }  // User defined constructor makes S non-POD.
+///     ~S() { } // User defined destructor makes it non-trivial.
+///   };
+///   void test() {
+///     const S &s_ref = S(); // Requires a CXXBindTemporaryExpr.
+///   }
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxBindTemporaryExpr()}
+/// matches the constructor call \match{S()}.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXBindTemporaryExpr>
     cxxBindTemporaryExpr;
 
 /// Matches nodes where temporaries are materialized.
@@ -1930,17 +2514,19 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXBindTemporaryExpr>
 ///   struct T {void func();};
 ///   T f();
 ///   void g(T);
+///   void foo() {
+///     T u(f());
+///     g(f());
+///     f().func();
+///     f(); // does not match
+///   }
 /// \endcode
-/// materializeTemporaryExpr() matches 'f()' in these statements
-/// \code
-///   T u(f());
-///   g(f());
-///   f().func();
-/// \endcode
-/// but does not match
-/// \code
-///   f();
-/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{materializeTemporaryExpr()} matches
+/// \match{std=c++14-or-earlier;count=3$f()} three times before C++17 and it
+/// matches \match{std=c++17-or-later$f()} one time with C++17 and later for
+/// \c f().func() , but it does not match the \nomatch{f()} in the last line in
+/// any version.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    MaterializeTemporaryExpr>
     materializeTemporaryExpr;
@@ -1949,20 +2535,34 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 ///
 /// Given
 /// \code
-///   new X;
+///   void* operator new(decltype(sizeof(void*)));
+///   struct X {};
+///   void foo() {
+///     auto* x = new X;
+///   }
 /// \endcode
-/// cxxNewExpr()
-///   matches 'new X'.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{cxxNewExpr()}
+/// matches \match{new X}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNewExpr> cxxNewExpr;
 
 /// Matches delete expressions.
 ///
 /// Given
 /// \code
-///   delete X;
+///   void* operator new(decltype(sizeof(void*)));
+///   void operator delete(void*);
+///   struct X {};
+///   void foo() {
+///     auto* x = new X;
+///     delete x;
+///   }
 /// \endcode
-/// cxxDeleteExpr()
-///   matches 'delete X'.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{cxxDeleteExpr()}
+/// matches \match{delete x}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXDeleteExpr>
     cxxDeleteExpr;
 
@@ -1976,13 +2576,16 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXDeleteExpr>
 ///   bool d() noexcept(noexcept(a()));
 ///   bool e = noexcept(b()) || noexcept(c());
 /// \endcode
-/// cxxNoexceptExpr()
-///   matches `noexcept(a())`, `noexcept(b())` and `noexcept(c())`.
-///   doesn't match the noexcept specifier in the declarations a, b, c or d.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{cxxNoexceptExpr()}
+/// matches \match{noexcept(a())}, \match{noexcept(b())} and
+/// \match{noexcept(c())}, but does not match the noexcept specifier in the
+/// declarations a, b, c or d.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNoexceptExpr>
     cxxNoexceptExpr;
 
-/// Matches a loop initializing the elements of an array in a number of contexts:
+/// Matches a loop initializing the elements of an array in a number of
+/// contexts:
 ///  * in the implicit copy/move constructor for a class with an array member
 ///  * when a lambda-expression captures an array by value
 ///  * when a decomposition declaration decomposes an array
@@ -1991,33 +2594,34 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNoexceptExpr>
 /// \code
 ///   void testLambdaCapture() {
 ///     int a[10];
-///     auto Lam1 = [a]() {
-///       return;
-///     };
+///     [a]() {};
 ///   }
 /// \endcode
-/// arrayInitLoopExpr() matches the implicit loop that initializes each element of
-/// the implicit array field inside the lambda object, that represents the array `a`
-/// captured by value.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{arrayInitLoopExpr()} matches the implicit loop that
+/// initializes each element of the implicit array field inside the lambda
+/// object, that represents the array \match{a} captured by value.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ArrayInitLoopExpr>
     arrayInitLoopExpr;
 
 /// The arrayInitIndexExpr consists of two subexpressions: a common expression
-/// (the source array) that is evaluated once up-front, and a per-element initializer
-/// that runs once for each array element. Within the per-element initializer,
-/// the current index may be obtained via an ArrayInitIndexExpr.
+/// (the source array) that is evaluated once up-front, and a per-element
+/// initializer that runs once for each array element. Within the per-element
+/// initializer, the current index may be obtained via an ArrayInitIndexExpr.
 ///
 /// Given
 /// \code
-///   void testStructBinding() {
+///   void testStructuredBinding() {
 ///     int a[2] = {1, 2};
 ///     auto [x, y] = a;
 ///   }
 /// \endcode
-/// arrayInitIndexExpr() matches the array index that implicitly iterates
-/// over the array `a` to copy each element to the anonymous array
-/// that backs the structured binding `[x, y]` elements of which are
-/// referred to by their aliases `x` and `y`.
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{type=none$arrayInitIndexExpr()} matches the array index
+/// that implicitly iterates over the array `a` to copy each element to the
+/// anonymous array that backs the structured binding.
+/// \match{}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ArrayInitIndexExpr>
     arrayInitIndexExpr;
 
@@ -2025,22 +2629,30 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ArrayInitIndexExpr>
 ///
 /// Given
 /// \code
-///   int i = a[1];
+///   void foo() {
+///     int a[2] = {0, 1};
+///     int i = a[1];
+///   }
 /// \endcode
-/// arraySubscriptExpr()
-///   matches "a[1]"
+/// The matcher \matcher{arraySubscriptExpr()}
+/// matches \match{a[1]}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ArraySubscriptExpr>
     arraySubscriptExpr;
 
 /// Matches the value of a default argument at the call site.
 ///
-/// Example matches the CXXDefaultArgExpr placeholder inserted for the
-///     default value of the second parameter in the call expression f(42)
-///     (matcher = cxxDefaultArgExpr())
+/// Given
 /// \code
 ///   void f(int x, int y = 0);
-///   f(42);
+///   void g() {
+///     f(42);
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{callExpr(has(cxxDefaultArgExpr()))}
+/// matches the \c CXXDefaultArgExpr placeholder inserted for the default value
+/// of the second parameter in the call expression \match{f(42)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXDefaultArgExpr>
     cxxDefaultArgExpr;
 
@@ -2051,13 +2663,18 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXDefaultArgExpr>
 /// Currently it does not match operators such as new delete.
 /// FIXME: figure out why these do not match?
 ///
-/// Example matches both operator<<((o << b), c) and operator<<(o, b)
-///     (matcher = cxxOperatorCallExpr())
+/// Given
 /// \code
+///   struct ostream;
 ///   ostream &operator<< (ostream &out, int i) { };
-///   ostream &o; int b = 1, c = 1;
-///   o << b << c;
+///   void f(ostream& o, int b, int c) {
+///     o << b << c;
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxOperatorCallExpr()} matches \match{o << b << c}
+/// and \match{o << b}.
 /// See also the binaryOperation() matcher for more-general matching of binary
 /// uses of this AST node.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXOperatorCallExpr>
@@ -2065,13 +2682,16 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXOperatorCallExpr>
 
 /// Matches C++17 fold expressions.
 ///
-/// Example matches `(0 + ... + args)`:
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
 ///       return (0 + ... + args);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr()} matches \match{(0 + ... + args)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXFoldExpr>
     cxxFoldExpr;
 
@@ -2079,17 +2699,21 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXFoldExpr>
 ///
 /// Example matches use of "<":
 /// \code
-///   #include <compare>
 ///   struct HasSpaceshipMem {
 ///     int a;
-///     constexpr auto operator<=>(const HasSpaceshipMem&) const = default;
+///     constexpr bool operator==(const HasSpaceshipMem&) const = default;
 ///   };
 ///   void compare() {
 ///     HasSpaceshipMem hs1, hs2;
-///     if (hs1 < hs2)
+///     if (hs1 != hs2)
 ///         return;
 ///   }
 /// \endcode
+/// \compile_args{-std=c++20-or-later}
+///
+/// The matcher \matcher{cxxRewrittenBinaryOperator()} matches
+/// \match{hs1 != hs2}.
+///
 /// See also the binaryOperation() matcher for more-general matching
 /// of this AST node.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
@@ -2098,25 +2722,34 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 
 /// Matches expressions.
 ///
-/// Example matches x()
+/// Given
 /// \code
-///   void f() { x(); }
+///   int f(int x, int y) { return x + y; }
 /// \endcode
+///
+/// The matcher \matcher{expr()} matches \match{x + y} once,
+/// \match{count=2$x} twice and \match{count=2$y} twice, matching the
+/// \c DeclRefExpr , and the \c ImplicitCastExpr that does an l- to r-value
+/// cast.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, Expr> expr;
 
 /// Matches expressions that refer to declarations.
 ///
-/// Example matches x in if (x)
+/// Given
 /// \code
-///   bool x;
-///   if (x) {}
+///   void f(bool x) {
+///     if (x) {}
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+///
+/// The matcher \matcher{declRefExpr()} matches \match{x}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DeclRefExpr>
     declRefExpr;
 
 /// Matches a reference to an ObjCIvar.
 ///
-/// Example: matches "a" in "init" method:
+/// Given
 /// \code
 /// @implementation A {
 ///   NSString *a;
@@ -2125,42 +2758,61 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, DeclRefExpr>
 ///   a = @"hello";
 /// }
 /// \endcode
+/// \compile_args{-ObjC}
+///
+/// The matcher \matcher{objcIvarRefExpr()} matches \match{a}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ObjCIvarRefExpr>
     objcIvarRefExpr;
 
 /// Matches a reference to a block.
 ///
-/// Example: matches "^{}":
+/// Given
 /// \code
 ///   void f() { ^{}(); }
 /// \endcode
+/// \compile_args{-ObjC}
+///
+/// The matcher \matcher{blockExpr()} matches \match{^{}}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, BlockExpr> blockExpr;
 
 /// Matches if statements.
 ///
-/// Example matches 'if (x) {}'
+/// Given
 /// \code
-///   if (x) {}
+///   void foo(int x) {
+///     if (x) {}
+///   }
 /// \endcode
+///
+/// The matcher \matcher{ifStmt()} matches \match{if (x) {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, IfStmt> ifStmt;
 
 /// Matches for statements.
 ///
-/// Example matches 'for (;;) {}'
+/// Given
 /// \code
-///   for (;;) {}
-///   int i[] =  {1, 2, 3}; for (auto a : i);
+///   void foo() {
+///     for (;;) {}
+///     int i[] =  {1, 2, 3}; for (auto a : i);
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{forStmt()} matches \match{for (;;) {}},
+/// but not \nomatch{for (auto a : i);}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ForStmt> forStmt;
 
 /// Matches the increment statement of a for loop.
 ///
-/// Example:
-///     forStmt(hasIncrement(unaryOperator(hasOperatorName("++"))))
-/// matches '++x' in
+/// Given
 /// \code
-///     for (x; x < N; ++x) { }
+/// void foo(int N) {
+///     for (int x = 0; x < N; ++x) { }
+/// }
 /// \endcode
+/// The matcher
+/// \matcher{forStmt(hasIncrement(unaryOperator(hasOperatorName("++"))))}
+/// matches \match{for (int x = 0; x < N; ++x) { }}
 AST_MATCHER_P(ForStmt, hasIncrement, internal::Matcher<Stmt>,
               InnerMatcher) {
   const Stmt *const Increment = Node.getInc();
@@ -2170,12 +2822,14 @@ AST_MATCHER_P(ForStmt, hasIncrement, internal::Matcher<Stmt>,
 
 /// Matches the initialization statement of a for loop.
 ///
-/// Example:
-///     forStmt(hasLoopInit(declStmt()))
-/// matches 'int x = 0' in
+/// Given
 /// \code
+/// void foo(int N) {
 ///     for (int x = 0; x < N; ++x) { }
+/// }
 /// \endcode
+/// The matcher \matcher{forStmt(hasLoopInit(declStmt()))}
+/// matches \match{for (int x = 0; x < N; ++x) { }}
 AST_MATCHER_P(ForStmt, hasLoopInit, internal::Matcher<Stmt>,
               InnerMatcher) {
   const Stmt *const Init = Node.getInit();
@@ -2184,22 +2838,31 @@ AST_MATCHER_P(ForStmt, hasLoopInit, internal::Matcher<Stmt>,
 
 /// Matches range-based for statements.
 ///
-/// cxxForRangeStmt() matches 'for (auto a : i)'
+/// Given
 /// \code
-///   int i[] =  {1, 2, 3}; for (auto a : i);
-///   for(int j = 0; j < 5; ++j);
+///   void foo() {
+///     int i[] =  {1, 2, 3}; for (auto a : i);
+///     for(int j = 0; j < 5; ++j);
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{cxxForRangeStmt()}
+/// matches \match{for (auto a : i);}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXForRangeStmt>
     cxxForRangeStmt;
 
 /// Matches the initialization statement of a for loop.
 ///
-/// Example:
-///     forStmt(hasLoopVariable(anything()))
-/// matches 'int x' in
+/// Given
 /// \code
+///   void foo() {
+///     int a[42] = {};
 ///     for (int x : a) { }
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{cxxForRangeStmt(hasLoopVariable(anything()))}
+/// matches \match{for (int x : a) { }}
 AST_MATCHER_P(CXXForRangeStmt, hasLoopVariable, internal::Matcher<VarDecl>,
               InnerMatcher) {
   const VarDecl *const Var = Node.getLoopVariable();
@@ -2208,12 +2871,16 @@ AST_MATCHER_P(CXXForRangeStmt, hasLoopVariable, internal::Matcher<VarDecl>,
 
 /// Matches the range initialization statement of a for loop.
 ///
-/// Example:
-///     forStmt(hasRangeInit(anything()))
-/// matches 'a' in
+/// Given
 /// \code
+///   void foo() {
+///     int a[42] = {};
 ///     for (int x : a) { }
+///   }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{cxxForRangeStmt(hasRangeInit(anything()))}
+/// matches \match{for (int x : a) { }}
 AST_MATCHER_P(CXXForRangeStmt, hasRangeInit, internal::Matcher<Expr>,
               InnerMatcher) {
   const Expr *const Init = Node.getRangeInit();
@@ -2224,40 +2891,52 @@ AST_MATCHER_P(CXXForRangeStmt, hasRangeInit, internal::Matcher<Expr>,
 ///
 /// Given
 /// \code
+/// void foo() {
 ///   while (true) {}
+/// }
 /// \endcode
-/// whileStmt()
-///   matches 'while (true) {}'.
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{whileStmt()}
+/// matches \match{while (true) {}}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, WhileStmt> whileStmt;
 
 /// Matches do statements.
 ///
 /// Given
 /// \code
+/// void foo() {
 ///   do {} while (true);
+/// }
 /// \endcode
-/// doStmt()
-///   matches 'do {} while(true)'
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{doStmt()}
+/// matches \match{do {} while (true)}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DoStmt> doStmt;
 
 /// Matches break statements.
 ///
 /// Given
 /// \code
+/// void foo() {
 ///   while (true) { break; }
+/// }
 /// \endcode
-/// breakStmt()
-///   matches 'break'
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{breakStmt()}
+/// matches \match{break}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, BreakStmt> breakStmt;
 
 /// Matches continue statements.
 ///
 /// Given
 /// \code
+/// void foo() {
 ///   while (true) { continue; }
+/// }
 /// \endcode
-/// continueStmt()
-///   matches 'continue'
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{continueStmt()}
+/// matches \match{continue}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ContinueStmt>
     continueStmt;
 
@@ -2265,10 +2944,46 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ContinueStmt>
 ///
 /// Given
 /// \code
-///   while (true) { co_return; }
+///   namespace std {
+///   template <typename T = void>
+///   struct coroutine_handle {
+///       static constexpr coroutine_handle from_address(void* addr) {
+///         return {};
+///       }
+///   };
+///
+///   struct always_suspend {
+///       bool await_ready() const noexcept;
+///       bool await_resume() const noexcept;
+///       template <typename T>
+///       bool await_suspend(coroutine_handle<T>) const noexcept;
+///   };
+///
+///   template <typename T>
+///   struct coroutine_traits {
+///       using promise_type = T::promise_type;
+///   };
+///   }  // namespace std
+///
+///   struct generator {
+///       struct promise_type {
+///           void return_value(int v);
+///           std::always_suspend yield_value(int&&);
+///           std::always_suspend initial_suspend() const noexcept;
+///           std::always_suspend final_suspend() const noexcept;
+///           void unhandled_exception();
+///           generator get_return_object();
+///       };
+///   };
+///
+///   generator f() {
+///       co_return 10;
+///   }
+///
 /// \endcode
-/// coreturnStmt()
-///   matches 'co_return'
+/// \compile_args{-std=c++20-or-later}
+/// The matcher \matcher{coreturnStmt(has(integerLiteral()))}
+/// matches \match{co_return 10}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoreturnStmt>
     coreturnStmt;
 
@@ -2276,44 +2991,55 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoreturnStmt>
 ///
 /// Given
 /// \code
+/// int foo() {
 ///   return 1;
+/// }
 /// \endcode
-/// returnStmt()
-///   matches 'return 1'
+/// The matcher \matcher{returnStmt()}
+/// matches \match{return 1}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ReturnStmt> returnStmt;
 
 /// Matches goto statements.
 ///
 /// Given
 /// \code
+/// void bar();
+/// void foo() {
 ///   goto FOO;
 ///   FOO: bar();
+/// }
 /// \endcode
-/// gotoStmt()
-///   matches 'goto FOO'
+/// The matcher \matcher{gotoStmt()}
+/// matches \match{goto FOO}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, GotoStmt> gotoStmt;
 
 /// Matches label statements.
 ///
 /// Given
 /// \code
+/// void bar();
+/// void foo() {
 ///   goto FOO;
 ///   FOO: bar();
+/// }
 /// \endcode
-/// labelStmt()
-///   matches 'FOO:'
+/// The matcher \matcher{labelStmt()}
+/// matches \match{FOO: bar()}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, LabelStmt> labelStmt;
 
 /// Matches address of label statements (GNU extension).
 ///
 /// Given
 /// \code
+/// void bar();
+/// void foo() {
 ///   FOO: bar();
 ///   void *ptr = &&FOO;
-///   goto *bar;
+///   goto *ptr;
+/// }
 /// \endcode
-/// addrLabelExpr()
-///   matches '&&FOO'
+/// The matcher \matcher{addrLabelExpr()}
+/// matches \match{&&FOO}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, AddrLabelExpr>
     addrLabelExpr;
 
@@ -2321,116 +3047,147 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, AddrLabelExpr>
 ///
 /// Given
 /// \code
+/// void foo(int a) {
 ///   switch(a) { case 42: break; default: break; }
+/// }
 /// \endcode
-/// switchStmt()
-///   matches 'switch(a)'.
+/// The matcher \matcher{switchStmt()}
+/// matches \match{switch(a) { case 42: break; default: break; }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, SwitchStmt> switchStmt;
 
 /// Matches case and default statements inside switch statements.
 ///
 /// Given
 /// \code
+/// void foo(int a) {
 ///   switch(a) { case 42: break; default: break; }
+/// }
 /// \endcode
-/// switchCase()
-///   matches 'case 42:' and 'default:'.
+/// The matcher \matcher{switchCase()}
+/// matches \match{case 42: break} and \match{default: break}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, SwitchCase> switchCase;
 
 /// Matches case statements inside switch statements.
 ///
 /// Given
 /// \code
+/// void foo(int a) {
 ///   switch(a) { case 42: break; default: break; }
+/// }
 /// \endcode
-/// caseStmt()
-///   matches 'case 42:'.
+/// The matcher \matcher{caseStmt()}
+/// matches \match{case 42: break}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CaseStmt> caseStmt;
 
 /// Matches default statements inside switch statements.
 ///
 /// Given
 /// \code
+/// void foo(int a) {
 ///   switch(a) { case 42: break; default: break; }
+/// }
 /// \endcode
-/// defaultStmt()
-///   matches 'default:'.
+/// The matcher \matcher{defaultStmt()}
+/// matches \match{default: break}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DefaultStmt>
     defaultStmt;
 
 /// Matches compound statements.
 ///
-/// Example matches '{}' and '{{}}' in 'for (;;) {{}}'
+/// Given
 /// \code
-///   for (;;) {{}}
+/// void foo() { for (;;) {{}} }
 /// \endcode
+///
+/// The matcher \matcher{compoundStmt()} matches
+/// \match{{ for (;;) {{}} }}, \match{{{}}} and \match{{}}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CompoundStmt>
     compoundStmt;
 
 /// Matches catch statements.
 ///
 /// \code
+/// void foo() {
 ///   try {} catch(int i) {}
+/// }
 /// \endcode
-/// cxxCatchStmt()
-///   matches 'catch(int i)'
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxCatchStmt()}
+/// matches \match{catch(int i) {}}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXCatchStmt>
     cxxCatchStmt;
 
 /// Matches try statements.
 ///
 /// \code
+/// void foo() {
 ///   try {} catch(int i) {}
+/// }
 /// \endcode
-/// cxxTryStmt()
-///   matches 'try {}'
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxTryStmt()}
+/// matches \match{try {} catch(int i) {}}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXTryStmt> cxxTryStmt;
 
 /// Matches throw expressions.
 ///
 /// \code
+/// void foo() {
 ///   try { throw 5; } catch(int i) {}
+/// }
 /// \endcode
-/// cxxThrowExpr()
-///   matches 'throw 5'
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxThrowExpr()}
+/// matches \match{throw 5}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXThrowExpr>
     cxxThrowExpr;
 
 /// Matches null statements.
 ///
 /// \code
+/// void foo() {
 ///   foo();;
+/// }
 /// \endcode
-/// nullStmt()
-///   matches the second ';'
+/// The matcher \matcher{nullStmt()}
+/// matches the second \match{;}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, NullStmt> nullStmt;
 
 /// Matches asm statements.
 ///
 /// \code
+/// void foo() {
 ///  int i = 100;
-///   __asm("mov al, 2");
+///   __asm("mov %al, 2");
+/// }
 /// \endcode
-/// asmStmt()
-///   matches '__asm("mov al, 2")'
+/// The matcher \matcher{asmStmt()}
+/// matches \match{__asm("mov %al, 2")}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, AsmStmt> asmStmt;
 
 /// Matches bool literals.
 ///
 /// Example matches true
 /// \code
-///   true
+///   bool Flag = true;
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+///
+/// The matcher \matcher{cxxBoolLiteral()} matches \match{true}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXBoolLiteralExpr>
     cxxBoolLiteral;
 
 /// Matches string literals (also matches wide string literals).
 ///
-/// Example matches "abcd", L"abcd"
+/// Given
 /// \code
 ///   char *s = "abcd";
 ///   wchar_t *ws = L"abcd";
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{stringLiteral()} matches \match{"abcd"} and
+/// \match{L"abcd"}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, StringLiteral>
     stringLiteral;
 
@@ -2439,11 +3196,15 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, StringLiteral>
 /// Not matching Hex-encoded chars (e.g. 0x1234, which is a IntegerLiteral),
 /// though.
 ///
-/// Example matches 'a', L'a'
+/// Given
 /// \code
 ///   char ch = 'a';
 ///   wchar_t chw = L'a';
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{characterLiteral()} matches \match{'a'} and
+/// \match{L'a'}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CharacterLiteral>
     characterLiteral;
 
@@ -2451,31 +3212,80 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CharacterLiteral>
 /// 1, 1L, 0x1 and 1U.
 ///
 /// Does not match character-encoded integers such as L'a'.
+///
+/// Given
+/// \code
+///   int a = 1;
+///   int b = 1L;
+///   int c = 0x1;
+///   int d = 1U;
+///   int e = 1.0;
+/// \endcode
+///
+/// The matcher \matcher{integerLiteral()} matches
+/// \match{1}, \match{1L}, \match{0x1} and \match{1U}, but does not match
+/// \nomatch{1.0}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, IntegerLiteral>
     integerLiteral;
 
 /// Matches float literals of all sizes / encodings, e.g.
 /// 1.0, 1.0f, 1.0L and 1e10.
 ///
-/// Does not match implicit conversions such as
+/// Given
 /// \code
-///   float a = 10;
+///   int a = 1.0;
+///   int b = 1.0F;
+///   int c = 1.0L;
+///   int d = 1e10;
+///   int e = 1;
 /// \endcode
+///
+/// The matcher \matcher{floatLiteral()} matches
+/// \match{1.0}, \match{1.0F}, \match{1.0L} and \match{1e10}, but does not match
+/// \nomatch{1}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, FloatingLiteral>
     floatLiteral;
 
 /// Matches imaginary literals, which are based on integer and floating
 /// point literals e.g.: 1i, 1.0i
+///
+/// Given
+/// \code
+///   auto a = 1i;
+///   auto b = 1.0i;
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{imaginaryLiteral()} matches \match{1i} and
+/// \match{1.0i}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ImaginaryLiteral>
     imaginaryLiteral;
 
 /// Matches fixed point literals
+///
+/// Given
+/// \code
+///   void f() {
+///     0.0k;
+///   }
+/// \endcode
+/// \compile_args{-ffixed-point}
+///
+/// The matcher \matcher{type=none$fixedPointLiteral()} matches \match{0.0k}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, FixedPointLiteral>
     fixedPointLiteral;
 
 /// Matches user defined literal operator call.
 ///
 /// Example match: "foo"_suffix
+/// Given
+/// \code
+///   float operator ""_foo(long double);
+///   float a = 1234.5_foo;
+/// \endcode
+/// \compile_args{-std=c++11}
+///
+/// The matcher \matcher{userDefinedLiteral()} matches \match{1234.5_foo}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, UserDefinedLiteral>
     userDefinedLiteral;
 
@@ -2483,9 +3293,12 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, UserDefinedLiteral>
 ///
 /// Example match: {1}, (1, 2)
 /// \code
-///   int array[4] = {1};
-///   vector int myvec = (vector int)(1, 2);
+///   struct vector { int x; int y; };
+///   struct vector myvec = (struct vector){ 1, 2 };
 /// \endcode
+///
+/// The matcher \matcher{compoundLiteralExpr()}
+/// matches \match{(struct vector){ 1, 2 }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CompoundLiteralExpr>
     compoundLiteralExpr;
 
@@ -2493,78 +3306,297 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CompoundLiteralExpr>
 ///
 /// Given
 /// \code
-///   co_await 1;
+///   namespace std {
+///   template <typename T = void>
+///   struct coroutine_handle {
+///       static constexpr coroutine_handle from_address(void* addr) {
+///         return {};
+///       }
+///   };
+///
+///   struct always_suspend {
+///       bool await_ready() const noexcept;
+///       bool await_resume() const noexcept;
+///       template <typename T>
+///       bool await_suspend(coroutine_handle<T>) const noexcept;
+///   };
+///
+///   template <typename T>
+///   struct coroutine_traits {
+///       using promise_type = T::promise_type;
+///   };
+///   }  // namespace std
+///
+///   struct generator {
+///       struct promise_type {
+///           std::always_suspend yield_value(int&&);
+///           std::always_suspend initial_suspend() const noexcept;
+///           std::always_suspend final_suspend() const noexcept;
+///           void return_void();
+///           void unhandled_exception();
+///           generator get_return_object();
+///       };
+///   };
+///
+///   std::always_suspend h();
+///
+///   generator g() { co_await h(); }
 /// \endcode
-/// coawaitExpr()
-///   matches 'co_await 1'
+/// \compile_args{-std=c++20-or-later}
+/// The matcher
+/// \matcher{coawaitExpr(has(callExpr(callee(functionDecl(hasName("h"))))))}
+/// matches \match{co_await h()}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoawaitExpr>
     coawaitExpr;
+
 /// Matches co_await expressions where the type of the promise is dependent
+///
+/// Given
+/// \code
+///   namespace std {
+///   template <typename T = void>
+///   struct coroutine_handle {
+///       static constexpr coroutine_handle from_address(void* addr) {
+///         return {};
+///       }
+///   };
+///
+///   struct always_suspend {
+///       bool await_ready() const noexcept;
+///       bool await_resume() const noexcept;
+///       template <typename T>
+///       bool await_suspend(coroutine_handle<T>) const noexcept;
+///   };
+///
+///   template <typename T>
+///   struct coroutine_traits {
+///       using promise_type = T::promise_type;
+///   };
+///   }  // namespace std
+///
+///   template <typename T>
+///   struct generator {
+///       struct promise_type {
+///           std::always_suspend yield_value(int&&);
+///           std::always_suspend initial_suspend() const noexcept;
+///           std::always_suspend final_suspend() const noexcept;
+///           void return_void();
+///           void unhandled_exception();
+///           generator get_return_object();
+///       };
+///   };
+///
+///   template <typename T>
+///   std::always_suspend h();
+///
+///   template <>
+///   std::always_suspend h<void>();
+///
+///   template<typename T>
+///   generator<T> g() { co_await h<T>(); }
+/// \endcode
+/// \compile_args{-std=c++20-or-later}
+/// The matcher \matcher{dependentCoawaitExpr()}
+/// matches \match{co_await h<T>()}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DependentCoawaitExpr>
     dependentCoawaitExpr;
+
 /// Matches co_yield expressions.
 ///
 /// Given
 /// \code
-///   co_yield 1;
+///   namespace std {
+///   template <typename T = void>
+///   struct coroutine_handle {
+///       static constexpr coroutine_handle from_address(void* addr) {
+///         return {};
+///       }
+///   };
+///
+///   struct always_suspend {
+///       bool await_ready() const noexcept;
+///       bool await_resume() const noexcept;
+///       template <typename T>
+///       bool await_suspend(coroutine_handle<T>) const noexcept;
+///   };
+///
+///   template <typename T>
+///   struct coroutine_traits {
+///       using promise_type = T::promise_type;
+///   };
+///   }  // namespace std
+///
+///   struct generator {
+///       struct promise_type {
+///           std::always_suspend yield_value(int&&);
+///           std::always_suspend initial_suspend() const noexcept;
+///           std::always_suspend final_suspend() const noexcept;
+///           void return_void();
+///           void unhandled_exception();
+///           generator get_return_object();
+///       };
+///   };
+///
+///   generator f() {
+///       while (true) {
+///           co_yield 10;
+///       }
+///   }
 /// \endcode
-/// coyieldExpr()
-///   matches 'co_yield 1'
+/// \compile_args{-std=c++20-or-later}
+/// The matcher \matcher{coyieldExpr()}
+/// matches \match{co_yield 10}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoyieldExpr>
     coyieldExpr;
 
 /// Matches coroutine body statements.
 ///
-/// coroutineBodyStmt() matches the coroutine below
+/// Given
 /// \code
-///   generator<int> gen() {
-///     co_return;
-///   }
+///   namespace std {
+///   template <typename T = void>
+///   struct coroutine_handle {
+///       static constexpr coroutine_handle from_address(void* addr) {
+///         return {};
+///       }
+///   };
+///
+///   struct suspend_always {
+///       bool await_ready() const noexcept;
+///       bool await_resume() const noexcept;
+///       template <typename T>
+///       bool await_suspend(coroutine_handle<T>) const noexcept;
+///   };
+///
+///   template <typename...>
+///   struct coroutine_traits {
+///       struct promise_type {
+///           std::suspend_always initial_suspend() const noexcept;
+///           std::suspend_always final_suspend() const noexcept;
+///           void return_void();
+///           void unhandled_exception();
+///           coroutine_traits get_return_object();
+///       };
+///   };
+///   }  // namespace std
+///
+///   void f() { while (true) { co_return; } }
+///
 /// \endcode
+/// \compile_args{-std=c++20-or-later}
+///
+/// The matcher \matcher{coroutineBodyStmt()} matches
+/// \match{{ while (true) { co_return; } }}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CoroutineBodyStmt>
     coroutineBodyStmt;
 
 /// Matches nullptr literal.
+///
+/// Given
+/// \code
+///   int a = 0;
+///   int* b = 0;
+///   int *c = nullptr;
+/// \endcode
+/// \compile_args{-std=c++11,c23-or-later}
+///
+/// The matcher \matcher{cxxNullPtrLiteralExpr()} matches \match{nullptr}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXNullPtrLiteralExpr>
     cxxNullPtrLiteralExpr;
 
 /// Matches GNU __builtin_choose_expr.
-extern const internal::VariadicDynCastAllOfMatcher<Stmt, ChooseExpr>
-    chooseExpr;
+///
+/// Given
+/// \code
+///   void f() { (void)__builtin_choose_expr(1, 2, 3); }
+/// \endcode
+///
+/// The matcher \matcher{chooseExpr()} matches
+/// \match{__builtin_choose_expr(1, 2, 3)}.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, ChooseExpr> chooseExpr;
 
 /// Matches builtin function __builtin_convertvector.
+///
+/// Given
+/// \code
+///   typedef double vector4double __attribute__((__vector_size__(32)));
+///   typedef float  vector4float  __attribute__((__vector_size__(16)));
+///   vector4float vf;
+///   void f() { (void)__builtin_convertvector(vf, vector4double); }
+/// \endcode
+///
+/// The matcher \matcher{convertVectorExpr()} matches
+/// \match{__builtin_convertvector(vf, vector4double)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ConvertVectorExpr>
     convertVectorExpr;
 
 /// Matches GNU __null expression.
+///
+/// Given
+/// \code
+///   auto val = __null;
+/// \endcode
+/// \compile_args{-std=c++11}
+///
+/// The matcher \matcher{gnuNullExpr()} matches \match{__null}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, GNUNullExpr>
     gnuNullExpr;
 
 /// Matches C11 _Generic expression.
+///
+/// Given
+/// \code
+///   double fdouble(double);
+///   float ffloat(float);
+///   #define GENERIC_MACRO(X) _Generic((X), double: fdouble, float: ffloat)(X)
+///
+///   void f() {
+///       GENERIC_MACRO(0.0);
+///       GENERIC_MACRO(0.0F);
+///   }
+/// \endcode
+/// \compile_args{-std=c}
+///
+/// The matcher \matcher{type=none$genericSelectionExpr()} matches
+/// the generic selection expression that is expanded in
+/// \match{GENERIC_MACRO(0.0)} and \match{GENERIC_MACRO(0.0F)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, GenericSelectionExpr>
     genericSelectionExpr;
 
 /// Matches atomic builtins.
-/// Example matches __atomic_load_n(ptr, 1)
+///
+/// Given
 /// \code
 ///   void foo() { int *ptr; __atomic_load_n(ptr, 1); }
 /// \endcode
+///
+/// The matcher \matcher{atomicExpr()} matches \match{__atomic_load_n(ptr, 1)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, AtomicExpr> atomicExpr;
 
 /// Matches statement expression (GNU extension).
 ///
-/// Example match: ({ int X = 4; X; })
+/// Given
 /// \code
-///   int C = ({ int X = 4; X; });
+///   void f() {
+///     int C = ({ int X = 4; X; });
+///   }
 /// \endcode
+///
+/// The matcher \matcher{stmtExpr()} matches \match{({ int X = 4; X; })}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, StmtExpr> stmtExpr;
 
 /// Matches binary operator expressions.
 ///
-/// Example matches a || b
+/// Given
 /// \code
-///   !(a || b)
+///   void foo(bool a, bool b) {
+///     !(a || b);
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+///
+/// The matcher \matcher{binaryOperator()} matches \match{a || b}.
+///
 /// See also the binaryOperation() matcher for more-general matching.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, BinaryOperator>
     binaryOperator;
@@ -2573,26 +3605,39 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, BinaryOperator>
 ///
 /// Example matches !a
 /// \code
-///   !a || b
+///   void foo(bool a, bool b) {
+///     !a || b;
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+///
+/// The matcher \matcher{unaryOperator()} matches \match{!a}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, UnaryOperator>
     unaryOperator;
 
 /// Matches conditional operator expressions.
 ///
-/// Example matches a ? b : c
+/// Given
 /// \code
-///   (a ? b : c) + 42
+///   int f(int a, int b, int c) {
+///     return (a ? b : c) + 42;
+///   }
 /// \endcode
+///
+/// The matcher \matcher{conditionalOperator()} matches \match{a ? b : c}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ConditionalOperator>
     conditionalOperator;
 
 /// Matches binary conditional operator expressions (GNU extension).
 ///
-/// Example matches a ?: b
+/// Given
 /// \code
-///   (a ?: b) + 42;
+///   int f(int a, int b) {
+///     return (a ?: b) + 42;
+///   }
 /// \endcode
+///
+/// The matcher \matcher{binaryConditionalOperator()} matches \match{a ?: b}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    BinaryConditionalOperator>
     binaryConditionalOperator;
@@ -2601,26 +3646,32 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 /// to reference another expressions and can be met
 /// in BinaryConditionalOperators, for example.
 ///
-/// Example matches 'a'
+/// Given
 /// \code
-///   (a ?: c) + 42;
+///   int f(int a, int b) {
+///     return (a ?: b) + 42;
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{opaqueValueExpr()} matches \match{count=2$a} twice,
+/// once for the check and once for the expression of the true path.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OpaqueValueExpr>
     opaqueValueExpr;
 
 /// Matches a C++ static_assert declaration.
 ///
-/// Example:
-///   staticAssertDecl()
-/// matches
-///   static_assert(sizeof(S) == sizeof(int))
-/// in
+/// Given
 /// \code
 ///   struct S {
 ///     int x;
 ///   };
 ///   static_assert(sizeof(S) == sizeof(int));
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{staticAssertDecl()}
+/// matches \match{static_assert(sizeof(S) == sizeof(int))}.
 extern const internal::VariadicDynCastAllOfMatcher<Decl, StaticAssertDecl>
     staticAssertDecl;
 
@@ -2634,6 +3685,10 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, StaticAssertDecl>
 /// \code
 ///   void* p = reinterpret_cast<char*>(&p);
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxReinterpretCastExpr()}
+/// matches \match{reinterpret_cast<char*>(&p)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXReinterpretCastExpr>
     cxxReinterpretCastExpr;
 
@@ -2642,49 +3697,56 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXReinterpretCastExpr>
 /// \see hasDestinationType
 /// \see reinterpretCast
 ///
-/// Example:
-///   cxxStaticCastExpr()
-/// matches
-///   static_cast<long>(8)
-/// in
+/// Given
 /// \code
 ///   long eight(static_cast<long>(8));
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxStaticCastExpr()}
+/// matches \match{static_cast<long>(8)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXStaticCastExpr>
     cxxStaticCastExpr;
 
 /// Matches a dynamic_cast expression.
 ///
-/// Example:
-///   cxxDynamicCastExpr()
-/// matches
-///   dynamic_cast<D*>(&b);
-/// in
+/// Given
 /// \code
 ///   struct B { virtual ~B() {} }; struct D : B {};
 ///   B b;
 ///   D* p = dynamic_cast<D*>(&b);
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxDynamicCastExpr()}
+/// matches \match{dynamic_cast<D*>(&b)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXDynamicCastExpr>
     cxxDynamicCastExpr;
 
 /// Matches a const_cast expression.
 ///
-/// Example: Matches const_cast<int*>(&r) in
+/// Given
 /// \code
 ///   int n = 42;
 ///   const int &r(n);
 ///   int* p = const_cast<int*>(&r);
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxConstCastExpr()}
+/// matches \match{const_cast<int*>(&r)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXConstCastExpr>
     cxxConstCastExpr;
 
 /// Matches a C-style cast expression.
 ///
-/// Example: Matches (int) 2.2f in
+/// Given
 /// \code
 ///   int i = (int) 2.2f;
 /// \endcode
+///
+/// The matcher \matcher{cStyleCastExpr()}
+/// matches \match{(int) 2.2f}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CStyleCastExpr>
     cStyleCastExpr;
 
@@ -2701,14 +3763,25 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CStyleCastExpr>
 ///
 /// \see hasDestinationType.
 ///
-/// Example: matches all five of the casts in
 /// \code
-///   int((int)(reinterpret_cast<int>(static_cast<int>(const_cast<int>(42)))))
-/// \endcode
-/// but does not match the implicit conversion in
-/// \code
-///   long ell = 42;
-/// \endcode
+///   struct S {};
+///   const S* s;
+///   S* s2 = const_cast<S*>(s);
+///
+///   const int val = 0;
+///   char val0 = val;
+///   char val1 = (char)val;
+///   char val2 = static_cast<char>(val);
+///   int* val3 = reinterpret_cast<int*>(val);
+///   char val4 = char(val);
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{explicitCastExpr()}
+/// matches \match{(char)val}, \match{static_cast<char>(val)},
+/// \match{reinterpret_cast<int*>(val)}, \match{const_cast<S*>(s)}
+/// and \match{char(val)}, but not the initialization of \c val0 with
+/// \nomatch{val}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ExplicitCastExpr>
     explicitCastExpr;
 
@@ -2716,41 +3789,89 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, ExplicitCastExpr>
 ///
 /// This matches many different places, including function call return value
 /// eliding, as well as any type conversions.
+///
+/// \code
+/// void f(int);
+/// void g(int val1, int val2) {
+///   unsigned int a = val1;
+///   f(val2);
+/// }
+/// \endcode
+///
+/// The matcher \matcher{implicitCastExpr()}
+/// matches \match{count=2$val1} for the implicit cast from an l- to an r-value
+/// and for the cast to \c{unsigned int}, \match{f} for the function pointer
+/// decay, and \match{val2} for the cast from an l- to an r-value.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, ImplicitCastExpr>
     implicitCastExpr;
 
 /// Matches any cast nodes of Clang's AST.
 ///
-/// Example: castExpr() matches each of the following:
-/// \code
-///   (int) 3;
-///   const_cast<Expr *>(SubExpr);
-///   char c = 0;
-/// \endcode
-/// but does not match
+/// Given
 /// \code
-///   int i = (0);
-///   int k = 0;
+///   struct S {};
+///   const S* s;
+///   S* s2 = const_cast<S*>(s);
+///
+///   const int val = 0;
+///   char val0 = 1;
+///   char val1 = (char)2;
+///   char val2 = static_cast<char>(3);
+///   int* val3 = reinterpret_cast<int*>(4);
+///   char val4 = char(5);
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{castExpr()}
+/// matches
+/// \match{const_cast<S*>(s)} and the implicit l- to r-value cast for \match{s},
+/// the implicit cast to \c char for the initializer \match{1},
+/// the c-style cast \match{(char)2} and it's implicit cast to \c char
+/// (part of the c-style cast) \match{2},
+/// \match{static_cast<char>(3)} and it's implicit cast to \c char
+/// (part of the \c static_cast) \match{3},
+/// \match{reinterpret_cast<int*>(4)},
+/// \match{char(5)} and it's implicit cast to \c char
+/// (part of the functional cast) \match{5}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CastExpr> castExpr;
 
 /// Matches functional cast expressions
 ///
-/// Example: Matches Foo(bar);
+/// Given
 /// \code
-///   Foo f = bar;
-///   Foo g = (Foo) bar;
-///   Foo h = Foo(bar);
+///   struct Foo {
+///     Foo(int x);
+///   };
+///
+///   void foo(int bar) {
+///     Foo f = bar;
+///     Foo g = (Foo) bar;
+///     Foo h = Foo(bar);
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxFunctionalCastExpr()}
+/// matches \match{Foo(bar)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXFunctionalCastExpr>
     cxxFunctionalCastExpr;
 
 /// Matches functional cast expressions having N != 1 arguments
 ///
-/// Example: Matches Foo(bar, bar)
+/// Given
 /// \code
-///   Foo h = Foo(bar, bar);
+///   struct Foo {
+///     Foo(int x, int y);
+///   };
+///
+///   void foo(int bar) {
+///     Foo h = Foo(bar, bar);
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxTemporaryObjectExpr()}
+/// matches \match{Foo(bar, bar)}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXTemporaryObjectExpr>
     cxxTemporaryObjectExpr;
 
@@ -2758,17 +3879,27 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CXXTemporaryObjectExpr>
 ///
 /// Example: Matches __func__
 /// \code
-///   printf("%s", __func__);
+///   void f() {
+///     const char* func_name = __func__;
+///   }
 /// \endcode
+///
+/// The matcher \matcher{predefinedExpr()}
+/// matches \match{__func__}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, PredefinedExpr>
     predefinedExpr;
 
 /// Matches C99 designated initializer expressions [C99 6.7.8].
 ///
-/// Example: Matches { [2].y = 1.0, [0].x = 1.0 }
+/// Example: Given
 /// \code
-///   point ptarray[10] = { [2].y = 1.0, [0].x = 1.0 };
+///   struct point2 { double x; double y; };
+///   struct point2 ptarray[10] = { [0].x = 1.0 };
+///   struct point2 pt = { .x = 2.0 };
 /// \endcode
+///
+/// The matcher \matcher{designatedInitExpr()}
+/// matches \match{[0].x = 1.0} and \match{.x = 2.0}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, DesignatedInitExpr>
     designatedInitExpr;
 
@@ -2777,23 +3908,53 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, DesignatedInitExpr>
 ///
 /// Example: Given
 /// \code
-///   point ptarray[10] = { [2].y = 1.0, [0].x = 1.0 };
-///   point ptarray2[10] = { [2].y = 1.0, [2].x = 0.0, [0].x = 1.0 };
+///   struct point2 { double x; double y; };
+///   struct point2 ptarray[10] = { [0].x = 1.0 };
+///   struct point2 pt = { .x = 2.0 };
 /// \endcode
-/// designatorCountIs(2)
-///   matches '{ [2].y = 1.0, [0].x = 1.0 }',
-///   but not '{ [2].y = 1.0, [2].x = 0.0, [0].x = 1.0 }'.
+///
+/// The matcher \matcher{designatedInitExpr(designatorCountIs(2))}
+/// matches \match{[0].x = 1.0}, but not \nomatch{.x = 2.0}.
 AST_MATCHER_P(DesignatedInitExpr, designatorCountIs, unsigned, N) {
   return Node.size() == N;
 }
 
 /// Matches \c QualTypes in the clang AST.
+///
+/// Given
+/// \code
+///   int a = 0;
+///   const int b = 1;
+/// \endcode
+///
+/// The matcher \matcher{varDecl(hasType(qualType(isConstQualified())))}
+/// matches \match{const int b = 1}, but not \nomatch{int a = 0}.
 extern const internal::VariadicAllOfMatcher<QualType> qualType;
 
 /// Matches \c Types in the clang AST.
+///
+/// Given
+/// \code
+///   const int b = 1;
+/// \endcode
+///
+/// The matcher \matcher{varDecl(hasType(type().bind("type")))}
+/// matches \match{const int b = 1}, with \matcher{type=sub$type()}
+/// matching \match{sub=type$int}.
 extern const internal::VariadicAllOfMatcher<Type> type;
 
 /// Matches \c TypeLocs in the clang AST.
+///
+/// That is, information about a type and where it was written.
+///
+/// \code
+///   void foo(int val);
+/// \endcode
+///
+/// The matcher \matcher{declaratorDecl(hasTypeLoc(typeLoc().bind("type")))}
+/// matches \match{void foo(int val)} and \match{int val}, with
+/// \matcher{type=sub$typeLoc()} matching \match{sub=type$void} and
+/// \match{sub=type$int} respectively.
 extern const internal::VariadicAllOfMatcher<TypeLoc> typeLoc;
 
 /// Matches if any of the given matchers matches.
@@ -2801,18 +3962,20 @@ extern const internal::VariadicAllOfMatcher<TypeLoc> typeLoc;
 /// Unlike \c anyOf, \c eachOf will generate a match result for each
 /// matching submatcher.
 ///
-/// For example, in:
-/// \code
-///   class A { int a; int b; };
-/// \endcode
-/// The matcher:
+/// Given
 /// \code
-///   cxxRecordDecl(eachOf(has(fieldDecl(hasName("a")).bind("v")),
-///                        has(fieldDecl(hasName("b")).bind("v"))))
+///   void f(int a, int b);
 /// \endcode
-/// will generate two results binding "v", the first of which binds
-/// the field declaration of \c a, the second the field declaration of
-/// \c b.
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{functionDecl(hasAnyParameter(
+/// eachOf(parmVarDecl(hasName("a")).bind("v"),
+///        parmVarDecl(hasName("b")).bind("v"))))}
+/// matches \match{void f(int a, int b)},
+/// with \matcher{type=sub$parmVarDecl(hasName("a"))} matching \match{sub=v$a}
+/// for one match,
+/// and with \matcher{type=sub$parmVarDecl(hasName("b"))} matching
+/// \match{sub=v$b} for the other match.
 ///
 /// Usable as: Any Matcher
 extern const internal::VariadicOperatorMatcherFunc<
@@ -2822,6 +3985,15 @@ extern const internal::VariadicOperatorMatcherFunc<
 /// Matches if any of the given matchers matches.
 ///
 /// Usable as: Any Matcher
+///
+/// \code
+///   char v0 = 'a';
+///   int v1 = 1;
+///   float v2 = 2.0;
+/// \endcode
+///
+/// The matcher \matcher{varDecl(anyOf(hasName("v0"), hasType(isInteger())))}
+/// matches \match{char v0 = 'a'} and \match{int v1 = 1}.
 extern const internal::VariadicOperatorMatcherFunc<
     2, std::numeric_limits<unsigned>::max()>
     anyOf;
@@ -2829,6 +4001,14 @@ extern const internal::VariadicOperatorMatcherFunc<
 /// Matches if all given matchers match.
 ///
 /// Usable as: Any Matcher
+///
+/// \code
+///   int v0 = 0;
+///   int v1 = 1;
+/// \endcode
+///
+/// The matcher \matcher{varDecl(allOf(hasName("v0"), hasType(isInteger())))}
+/// matches \match{int v0 = 0}.
 extern const internal::VariadicOperatorMatcherFunc<
     2, std::numeric_limits<unsigned>::max()>
     allOf;
@@ -2839,22 +4019,14 @@ extern const internal::VariadicOperatorMatcherFunc<
 /// Useful when additional information which may or may not present about a main
 /// matching node is desired.
 ///
-/// For example, in:
-/// \code
-///   class Foo {
-///     int bar;
-///   }
-/// \endcode
-/// The matcher:
+/// Given
 /// \code
-///   cxxRecordDecl(
-///     optionally(has(
-///       fieldDecl(hasName("bar")).bind("var")
-///   ))).bind("record")
+///   int a = 0;
+///   int b;
 /// \endcode
-/// will produce a result binding for both "record" and "var".
-/// The matcher will produce a "record" binding for even if there is no data
-/// member named "bar" in that class.
+///
+/// The matcher \matcher{varDecl(optionally(hasInitializer(expr())))}
+/// matches \match{int a = 0} and \match{int b}.
 ///
 /// Usable as: Any Matcher
 extern const internal::VariadicOperatorMatcherFunc<1, 1> optionally;
@@ -2863,11 +4035,12 @@ extern const internal::VariadicOperatorMatcherFunc<1, 1> optionally;
 ///
 /// Given
 /// \code
-///   Foo x = bar;
+///   int x = 42;
 ///   int y = sizeof(x) + alignof(x);
 /// \endcode
-/// unaryExprOrTypeTraitExpr()
-///   matches \c sizeof(x) and \c alignof(x)
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+/// The matcher \matcher{unaryExprOrTypeTraitExpr()}
+/// matches \match{sizeof(x)} and \match{alignof(x)}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt,
                                                    UnaryExprOrTypeTraitExpr>
     unaryExprOrTypeTraitExpr;
@@ -2876,26 +4049,26 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 ///
 /// Given
 /// \code
-///   if (true);
-///   for (; true; );
-/// \endcode
-/// with the matcher
-/// \code
-///   mapAnyOf(ifStmt, forStmt).with(
-///     hasCondition(cxxBoolLiteralExpr(equals(true)))
-///     ).bind("trueCond")
-/// \endcode
-/// matches the \c if and the \c for. It is equivalent to:
-/// \code
-///   auto trueCond = hasCondition(cxxBoolLiteralExpr(equals(true)));
-///   anyOf(
-///     ifStmt(trueCond).bind("trueCond"),
-///     forStmt(trueCond).bind("trueCond")
-///     );
+///   void f() {
+///     if (true);
+///     for (; true; );
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+///
+/// The matcher \matcher{stmt(mapAnyOf(ifStmt, forStmt).with(
+///     hasCondition(cxxBoolLiteral(equals(true)))
+///     ))},
+/// which is equivalent to
+/// \matcher{stmt(anyOf(
+///     ifStmt(hasCondition(cxxBoolLiteral(equals(true)))).bind("trueCond"),
+///     forStmt(hasCondition(cxxBoolLiteral(equals(true)))).bind("trueCond")
+///     ))},
+/// matches \match{if (true);} and \match{for (; true; );}.
 ///
 /// The with() chain-call accepts zero or more matchers which are combined
 /// as-if with allOf() in each of the node matchers.
+///
 /// Usable as: Any Matcher
 template <typename T, typename... U>
 auto mapAnyOf(internal::VariadicDynCastAllOfMatcher<T, U> const &...) {
@@ -2904,34 +4077,23 @@ auto mapAnyOf(internal::VariadicDynCastAllOfMatcher<T, U> const &...) {
 
 /// Matches nodes which can be used with binary operators.
 ///
-/// The code
-/// \code
-///   var1 != var2;
-/// \endcode
-/// might be represented in the clang AST as a binaryOperator, a
-/// cxxOperatorCallExpr or a cxxRewrittenBinaryOperator, depending on
+/// A comparison of two expressions might be represented in the clang AST as a
+/// \c binaryOperator, a \c cxxOperatorCallExpr or a
+/// \c cxxRewrittenBinaryOperator, depending on
 ///
 /// * whether the types of var1 and var2 are fundamental (binaryOperator) or at
-///   least one is a class type (cxxOperatorCallExpr)
+///   least one is a class type (\c cxxOperatorCallExpr)
 /// * whether the code appears in a template declaration, if at least one of the
-///   vars is a dependent-type (binaryOperator)
+///   vars is a dependent-type (\c binaryOperator)
 /// * whether the code relies on a rewritten binary operator, such as a
 /// spaceship operator or an inverted equality operator
-/// (cxxRewrittenBinaryOperator)
+/// (\c cxxRewrittenBinaryOperator)
 ///
 /// This matcher elides details in places where the matchers for the nodes are
 /// compatible.
 ///
 /// Given
 /// \code
-///   binaryOperation(
-///     hasOperatorName("!="),
-///     hasLHS(expr().bind("lhs")),
-///     hasRHS(expr().bind("rhs"))
-///   )
-/// \endcode
-/// matches each use of "!=" in:
-/// \code
 ///   struct S{
 ///       bool operator!=(const S&) const;
 ///   };
@@ -2945,25 +4107,28 @@ auto mapAnyOf(internal::VariadicDynCastAllOfMatcher<T, U> const &...) {
 ///   template<typename T>
 ///   void templ()
 ///   {
-///      1 != 2;
+///      3 != 4;
 ///      T() != S();
 ///   }
 ///   struct HasOpEq
 ///   {
-///       bool operator==(const HasOpEq &) const;
+///       friend bool
+///       operator==(const HasOpEq &, const HasOpEq&) noexcept = default;
 ///   };
 ///
 ///   void inverse()
 ///   {
-///       HasOpEq s1;
-///       HasOpEq s2;
-///       if (s1 != s2)
+///       HasOpEq e1;
+///       HasOpEq e2;
+///       if (e1 != e2)
 ///           return;
 ///   }
 ///
 ///   struct HasSpaceship
 ///   {
-///       bool operator<=>(const HasOpEq &) const;
+///       friend bool
+///       operator<=>(const HasSpaceship &,
+///                   const HasSpaceship&) noexcept = default;
 ///   };
 ///
 ///   void use_spaceship()
@@ -2974,18 +4139,27 @@ auto mapAnyOf(internal::VariadicDynCastAllOfMatcher<T, U> const &...) {
 ///           return;
 ///   }
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++20-or-later}
+///
+/// The matcher \matcher{binaryOperation(
+///     hasOperatorName("!="),
+///     hasLHS(expr().bind("lhs")),
+///     hasRHS(expr().bind("rhs"))
+///   )}
+/// matches \match{1 != 2}, \match{S() != S()}, \match{3 != 4},
+/// \match{T() != S()}, \match{e1 != e2} and \match{s1 != s2}.
 extern const internal::MapAnyOfMatcher<BinaryOperator, CXXOperatorCallExpr,
                                        CXXRewrittenBinaryOperator>
     binaryOperation;
 
 /// Matches function calls and constructor calls
 ///
-/// Because CallExpr and CXXConstructExpr do not share a common
+/// Because \c CallExpr and \c CXXConstructExpr do not share a common
 /// base class with API accessing arguments etc, AST Matchers for code
 /// which should match both are typically duplicated. This matcher
 /// removes the need for duplication.
 ///
-/// Given code
+/// Given
 /// \code
 /// struct ConstructorTakesInt
 /// {
@@ -3006,12 +4180,12 @@ extern const internal::MapAnyOfMatcher<BinaryOperator, CXXOperatorCallExpr,
 ///   ConstructorTakesInt cti(42);
 /// }
 /// \endcode
+/// \compile_args{-std=c++}
 ///
 /// The matcher
-/// \code
-/// invocation(hasArgument(0, integerLiteral(equals(42))))
-/// \endcode
-/// matches the expression in both doCall and doConstruct
+/// \matcher{expr(invocation(hasArgument(0, integerLiteral(equals(42)))))}
+/// matches the expressions \match{callTakesInt(42)}
+/// and \match{cti(42)}.
 extern const internal::MapAnyOfMatcher<CallExpr, CXXConstructExpr> invocation;
 
 /// Matches unary expressions that have a specific type of argument.
@@ -3020,8 +4194,10 @@ extern const internal::MapAnyOfMatcher<CallExpr, CXXConstructExpr> invocation;
 /// \code
 ///   int a, c; float b; int s = sizeof(a) + sizeof(b) + alignof(c);
 /// \endcode
-/// unaryExprOrTypeTraitExpr(hasArgumentOfType(asString("int"))
-///   matches \c sizeof(a) and \c alignof(c)
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+/// The matcher
+/// \matcher{unaryExprOrTypeTraitExpr(hasArgumentOfType(asString("int")))}
+/// matches \match{sizeof(a)} and \match{alignof(c)}
 AST_MATCHER_P(UnaryExprOrTypeTraitExpr, hasArgumentOfType,
               internal::Matcher<QualType>, InnerMatcher) {
   const QualType ArgumentType = Node.getTypeOfArgument();
@@ -3033,10 +4209,11 @@ AST_MATCHER_P(UnaryExprOrTypeTraitExpr, hasArgumentOfType,
 /// Given
 /// \code
 ///   int x;
-///   int s = sizeof(x) + alignof(x)
+///   int s = sizeof(x) + alignof(x);
 /// \endcode
-/// unaryExprOrTypeTraitExpr(ofKind(UETT_SizeOf))
-///   matches \c sizeof(x)
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+/// The matcher \matcher{unaryExprOrTypeTraitExpr(ofKind(UETT_SizeOf))}
+/// matches \match{sizeof(x)}
 ///
 /// If the matcher is use from clang-query, UnaryExprOrTypeTrait parameter
 /// should be passed as a quoted string. e.g., ofKind("UETT_SizeOf").
@@ -3046,8 +4223,17 @@ AST_MATCHER_P(UnaryExprOrTypeTraitExpr, ofKind, UnaryExprOrTypeTrait, Kind) {
 
 /// Same as unaryExprOrTypeTraitExpr, but only matching
 /// alignof.
-inline internal::BindableMatcher<Stmt> alignOfExpr(
-    const internal::Matcher<UnaryExprOrTypeTraitExpr> &InnerMatcher) {
+///
+/// Given
+/// \code
+///   int align = alignof(int);
+/// \endcode
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+///
+/// The matcher \matcher{alignOfExpr(expr())}
+/// matches \match{alignof(int)}.
+inline internal::BindableMatcher<Stmt>
+alignOfExpr(const internal::Matcher<UnaryExprOrTypeTraitExpr> &InnerMatcher) {
   return stmt(unaryExprOrTypeTraitExpr(
       allOf(anyOf(ofKind(UETT_AlignOf), ofKind(UETT_PreferredAlignOf)),
             InnerMatcher)));
@@ -3055,6 +4241,15 @@ inline internal::BindableMatcher<Stmt> alignOfExpr(
 
 /// Same as unaryExprOrTypeTraitExpr, but only matching
 /// sizeof.
+///
+/// Given
+/// \code
+///   struct S { double x; double y; };
+///   int size = sizeof(struct S);
+/// \endcode
+///
+/// The matcher \matcher{sizeOfExpr(expr())}
+/// matches \match{sizeof(struct S)}.
 inline internal::BindableMatcher<Stmt> sizeOfExpr(
     const internal::Matcher<UnaryExprOrTypeTraitExpr> &InnerMatcher) {
   return stmt(unaryExprOrTypeTraitExpr(
@@ -3067,15 +4262,26 @@ inline internal::BindableMatcher<Stmt> sizeOfExpr(
 /// with '<enclosing>::'.
 /// Does not match typedefs of an underlying type with the given name.
 ///
-/// Example matches X (Name == "X")
+/// Given
 /// \code
 ///   class X;
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// Example matches X (Name is one of "::a::b::X", "a::b::X", "b::X", "X")
+/// The matcher \matcher{namedDecl(hasName("X"))}
+/// matches \match{class X}.
+///
+/// Given
 /// \code
 ///   namespace a { namespace b { class X; } }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matchers \matcher{namedDecl(hasName("::a::b::X"))},
+/// \matcher{namedDecl(hasName("a::b::X"))},
+/// \matcher{namedDecl(hasName("b::X"))} and
+/// \matcher{namedDecl(hasName("X"))}
+/// match \match{class X}.
 inline internal::Matcher<NamedDecl> hasName(StringRef Name) {
   return internal::Matcher<NamedDecl>(
       new internal::HasNameMatcher({std::string(Name)}));
@@ -3084,13 +4290,17 @@ inline internal::Matcher<NamedDecl> hasName(StringRef Name) {
 /// Matches NamedDecl nodes that have any of the specified names.
 ///
 /// This matcher is only provided as a performance optimization of hasName.
+///
+/// Given
 /// \code
-///     hasAnyName(a, b, c)
-/// \endcode
-///  is equivalent to, but faster than
-/// \code
-///     anyOf(hasName(a), hasName(b), hasName(c))
+///   void f(int a, int b);
 /// \endcode
+///
+/// The matcher \matcher{namedDecl(hasAnyName("a", "b"))},
+/// which is equivalent to the matcher
+/// \matcher{namedDecl(hasAnyName("a", "b"))},
+/// matches \match{int a} and \match{int b}, but not
+/// \nomatch{void f(int a, int b)}.
 extern const internal::VariadicFunction<internal::Matcher<NamedDecl>, StringRef,
                                         internal::hasAnyNameFunc>
     hasAnyName;
@@ -3102,15 +4312,14 @@ extern const internal::VariadicFunction<internal::Matcher<NamedDecl>, StringRef,
 /// prefixing the name with '<enclosing>::'.  Does not match typedefs
 /// of an underlying type with the given name.
 ///
-/// Example matches X (regexp == "::X")
-/// \code
-///   class X;
-/// \endcode
-///
-/// Example matches X (regexp is one of "::X", "^foo::.*X", among others)
+/// Given
 /// \code
 ///   namespace foo { namespace bar { class X; } }
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{namedDecl(matchesName("^::foo:.*X"))}
+/// matches \match{class X}.
 AST_MATCHER_REGEX(NamedDecl, matchesName, RegExp) {
   std::string FullNameString = "::" + Node.getQualifiedNameAsString();
   return RegExp->match(FullNameString);
@@ -3121,18 +4330,21 @@ AST_MATCHER_REGEX(NamedDecl, matchesName, RegExp) {
 /// Matches overloaded operator names specified in strings without the
 /// "operator" prefix: e.g. "<<".
 ///
-/// Given:
+/// Given
 /// \code
-///   class A { int operator*(); };
+///   struct A { int operator*(); };
 ///   const A &operator<<(const A &a, const A &b);
-///   A a;
-///   a << a;   // <-- This matches
+///   void f(A a) {
+///     a << a;   // <-- This matches
+///   }
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// \c cxxOperatorCallExpr(hasOverloadedOperatorName("<<"))) matches the
-/// specified line and
-/// \c cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")))
-/// matches the declaration of \c A.
+/// The matcher \matcher{cxxOperatorCallExpr(hasOverloadedOperatorName("<<"))}
+/// matches \match{a << a}.
+/// The matcher
+/// \matcher{cxxRecordDecl(hasMethod(hasOverloadedOperatorName("*")))}
+/// matches \match{struct A { int operator*(); }}.
 ///
 /// Usable as: Matcher<CXXOperatorCallExpr>, Matcher<FunctionDecl>
 inline internal::PolymorphicMatcher<
@@ -3152,6 +4364,32 @@ hasOverloadedOperatorName(StringRef Name) {
 /// "operator" prefix: e.g. "<<".
 ///
 ///   hasAnyOverloadedOperatorName("+", "-")
+///
+/// Given
+/// \code
+///   struct Point { double x; double y; };
+///   Point operator+(const Point&, const Point&);
+///   Point operator-(const Point&, const Point&);
+///
+///   Point sub(Point a, Point b) {
+///     return b - a;
+///   }
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{functionDecl(hasAnyOverloadedOperatorName("+", "-"))},
+/// which is equivalent to
+/// \matcher{functionDecl(anyOf(hasAnyOverloadedOperatorName("+"),
+/// hasOverloadedOperatorName("-")))},
+/// matches \match{Point operator+(const Point&, const Point&)} and
+/// \match{Point operator-(const Point&, const Point&)}.
+/// The matcher
+/// \matcher{cxxOperatorCallExpr(hasAnyOverloadedOperatorName("+", "-"))},
+/// which is equivalent to
+/// \matcher{cxxOperatorCallExpr(anyOf(hasOverloadedOperatorName("+"),
+/// hasOverloadedOperatorName("-")))},
+/// matches \match{b - a}.
+///
 /// Is equivalent to
 ///   anyOf(hasOverloadedOperatorName("+"), hasOverloadedOperatorName("-"))
 extern const internal::VariadicFunction<
@@ -3181,7 +4419,9 @@ extern const internal::VariadicFunction<
 ///       s.mem();
 ///   }
 /// \endcode
-/// \c cxxDependentScopeMemberExpr(hasMemberName("mem")) matches `s.mem()`
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxDependentScopeMemberExpr(hasMemberName("mem"))}
+/// matches \match{s.mem}.
 AST_MATCHER_P(CXXDependentScopeMemberExpr, hasMemberName, std::string, N) {
   return Node.getMember().getAsString() == N;
 }
@@ -3207,19 +4447,20 @@ AST_MATCHER_P(CXXDependentScopeMemberExpr, hasMemberName, std::string, N) {
 ///       s.mem();
 ///   }
 /// \endcode
-/// The matcher
-/// @code
-/// \c cxxDependentScopeMemberExpr(
-///   hasObjectExpression(declRefExpr(hasType(templateSpecializationType(
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxDependentScopeMemberExpr(
+///   hasObjectExpression(declRefExpr(hasType(
+///     elaboratedType(namesType(templateSpecializationType(
 ///       hasDeclaration(classTemplateDecl(has(cxxRecordDecl(has(
 ///           cxxMethodDecl(hasName("mem")).bind("templMem")
 ///           )))))
-///       )))),
+///     )))
+///   ))),
 ///   memberHasSameNameAsBoundNode("templMem")
-///   )
-/// @endcode
-/// first matches and binds the @c mem member of the @c S template, then
-/// compares its name to the usage in @c s.mem() in the @c x function template
+/// )}
+/// matches \match{s.mem}, with the inner matcher
+/// \matcher{type=sub$cxxMethodDecl(hasName("mem"))} matching
+/// \match{sub=templMem$void mem()} of the \c S template.
 AST_MATCHER_P(CXXDependentScopeMemberExpr, memberHasSameNameAsBoundNode,
               std::string, BindingID) {
   auto MemberName = Node.getMember().getAsString();
@@ -3244,26 +4485,33 @@ AST_MATCHER_P(CXXDependentScopeMemberExpr, memberHasSameNameAsBoundNode,
 ///
 /// Example matches Y, Z, C (Base == hasName("X"))
 /// \code
-///   class X;
+///   class X {};
 ///   class Y : public X {};  // directly derived
 ///   class Z : public Y {};  // indirectly derived
 ///   typedef X A;
 ///   typedef A B;
 ///   class C : public B {};  // derived from a typedef of X
-/// \endcode
 ///
-/// In the following example, Bar matches isDerivedFrom(hasName("X")):
-/// \code
-///   class Foo;
-///   typedef Foo X;
-///   class Bar : public Foo {};  // derived from a type that X is a typedef of
+///   class Foo {};
+///   typedef Foo Alias;
+///   class Bar : public Alias {};
+///   // derived from a type that Alias is a typedef of Foo
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxRecordDecl(isDerivedFrom(hasName("X")))}
+/// matches \match{class Y : public X {}}, \match{class Z : public Y {}}
+/// and \match{class C : public B {}}.
+///
+/// The matcher \matcher{cxxRecordDecl(isDerivedFrom(hasName("Foo")))}
+/// matches \match{class Bar : public Alias {}}.
 ///
 /// In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
 /// \code
 ///   @interface NSObject @end
 ///   @interface Bar : NSObject @end
 /// \endcode
+/// \compile_args{-ObjC}
 ///
 /// Usable as: Matcher<CXXRecordDecl>, Matcher<ObjCInterfaceDecl>
 AST_POLYMORPHIC_MATCHER_P(
@@ -3281,6 +4529,44 @@ AST_POLYMORPHIC_MATCHER_P(
 }
 
 /// Overloaded method as shortcut for \c isDerivedFrom(hasName(...)).
+///
+/// Matches C++ classes that are directly or indirectly derived from a class
+/// matching \c Base, or Objective-C classes that directly or indirectly
+/// subclass a class matching \c Base.
+///
+/// Note that a class is not considered to be derived from itself.
+///
+/// Example matches Y, Z, C (Base == hasName("X"))
+/// \code
+///   class X {};
+///   class Y : public X {};  // directly derived
+///   class Z : public Y {};  // indirectly derived
+///   typedef X A;
+///   typedef A B;
+///   class C : public B {};  // derived from a typedef of X
+///
+///   class Foo {};
+///   typedef Foo Alias;
+///   class Bar : public Alias {};  // derived from Alias, which is a
+///                                 // typedef of Foo
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxRecordDecl(isDerivedFrom("X"))}
+/// matches \match{class Y : public X {}}, \match{class Z : public Y {}}
+/// and \match{class C : public B {}}.
+///
+/// The matcher \matcher{cxxRecordDecl(isDerivedFrom("Foo"))}
+/// matches \match{class Bar : public Alias {}}.
+///
+/// In the following example, Bar matches isDerivedFrom(hasName("NSObject"))
+/// \code
+///   @interface NSObject @end
+///   @interface Bar : NSObject @end
+/// \endcode
+/// \compile_args{-ObjC}
+///
+/// Usable as: Matcher<CXXRecordDecl>, Matcher<ObjCInterfaceDecl>
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     isDerivedFrom,
     AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl, ObjCInterfaceDecl),
@@ -3300,17 +4586,21 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 /// Matches C++ classes that have a direct or indirect base matching \p
 /// BaseSpecMatcher.
 ///
-/// Example:
-/// matcher hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
+/// Given
 /// \code
-///   class Foo;
+///   class Foo {};
 ///   class Bar : Foo {};
 ///   class Baz : Bar {};
-///   class SpecialBase;
+///   class SpecialBase {};
 ///   class Proxy : SpecialBase {};  // matches Proxy
 ///   class IndirectlyDerived : Proxy {};  //matches IndirectlyDerived
 /// \endcode
+/// \compile_args{-std=c++}
 ///
+/// The matcher
+/// \matcher{cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("SpecialBase")))))}
+/// matches \match{class Proxy : SpecialBase {}} and
+/// \match{class IndirectlyDerived : Proxy {}}.
 // FIXME: Refactor this and isDerivedFrom to reuse implementation.
 AST_MATCHER_P(CXXRecordDecl, hasAnyBase, internal::Matcher<CXXBaseSpecifier>,
               BaseSpecMatcher) {
@@ -3319,16 +4609,19 @@ AST_MATCHER_P(CXXRecordDecl, hasAnyBase, internal::Matcher<CXXBaseSpecifier>,
 
 /// Matches C++ classes that have a direct base matching \p BaseSpecMatcher.
 ///
-/// Example:
-/// matcher hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase"))))
+/// Given
 /// \code
-///   class Foo;
+///   class Foo {};
 ///   class Bar : Foo {};
 ///   class Baz : Bar {};
-///   class SpecialBase;
+///   class SpecialBase {};
 ///   class Proxy : SpecialBase {};  // matches Proxy
 ///   class IndirectlyDerived : Proxy {};  // doesn't match
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasDirectBase(hasType(cxxRecordDecl(hasName("SpecialBase")))))}
+/// matches \match{class Proxy : SpecialBase {}}.
 AST_MATCHER_P(CXXRecordDecl, hasDirectBase, internal::Matcher<CXXBaseSpecifier>,
               BaseSpecMatcher) {
   return Node.hasDefinition() &&
@@ -3339,6 +4632,22 @@ AST_MATCHER_P(CXXRecordDecl, hasDirectBase, internal::Matcher<CXXBaseSpecifier>,
 
 /// Similar to \c isDerivedFrom(), but also matches classes that directly
 /// match \c Base.
+///
+/// Given
+/// \code
+///   class X {};
+///   class Y : public X {};  // directly derived
+///   class Z : public Y {};  // indirectly derived
+///   typedef X A;
+///   typedef A B;
+///   class C : public B {};  // derived from a typedef of X
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(isSameOrDerivedFrom(cxxRecordDecl(hasName("X"))),
+/// isDefinition())}
+/// matches \match{class X {}}, \match{class Y : public X {}},
+/// \match{class Z : public Y {}} and \match{class C : public B {}}.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     isSameOrDerivedFrom,
     AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl, ObjCInterfaceDecl),
@@ -3352,8 +4661,25 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
   return Matcher<ObjCInterfaceDecl>(M).matches(*InterfaceDecl, Finder, Builder);
 }
 
+/// Similar to \c isDerivedFrom(), but also matches classes that directly
+/// match \c Base.
 /// Overloaded method as shortcut for
 /// \c isSameOrDerivedFrom(hasName(...)).
+///
+/// Given
+/// \code
+///   class X {};
+///   class Y : public X {};  // directly derived
+///   class Z : public Y {};  // indirectly derived
+///   typedef X A;
+///   typedef A B;
+///   class C : public B {};  // derived from a typedef of X
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(isSameOrDerivedFrom("X"), isDefinition())}
+/// matches \match{class X {}}, \match{class Y : public X {}},
+/// \match{class Z : public Y {}} and \match{class C : public B {}}.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     isSameOrDerivedFrom,
     AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl, ObjCInterfaceDecl),
@@ -3375,22 +4701,30 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 ///
 /// Note that a class is not considered to be derived from itself.
 ///
-/// Example matches Y, C (Base == hasName("X"))
+/// Given
 /// \code
-///   class X;
+///   class X {};
 ///   class Y : public X {};  // directly derived
 ///   class Z : public Y {};  // indirectly derived
 ///   typedef X A;
 ///   typedef A B;
 ///   class C : public B {};  // derived from a typedef of X
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(isDirectlyDerivedFrom(namedDecl(hasName("X"))))}
+/// matches \match{class Y : public X {}} and \match{class C : public B {}}
+/// (Base == hasName("X").
 ///
 /// In the following example, Bar matches isDerivedFrom(hasName("X")):
 /// \code
-///   class Foo;
+///   class Foo {};
 ///   typedef Foo X;
 ///   class Bar : public Foo {};  // derived from a type that X is a typedef of
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(isDerivedFrom(hasName("X")))}
+/// matches \match{class Bar : public Foo {}}.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     isDirectlyDerivedFrom,
     AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl, ObjCInterfaceDecl),
@@ -3406,6 +4740,18 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 }
 
 /// Overloaded method as shortcut for \c isDirectlyDerivedFrom(hasName(...)).
+///
+/// Given
+/// \code
+///   struct Base {};
+///   struct DirectlyDerived : public Base {};
+///   struct IndirectlyDerived : public DirectlyDerived {};
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{cxxRecordDecl(isDirectlyDerivedFrom("Base"))}
+/// matches \match{struct DirectlyDerived : public Base {}}, but not
+/// \nomatch{struct IndirectlyDerived : public DirectlyDerived {}}.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     isDirectlyDerivedFrom,
     AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl, ObjCInterfaceDecl),
@@ -3423,14 +4769,16 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 /// Matches the first method of a class or struct that satisfies \c
 /// InnerMatcher.
 ///
-/// Given:
+/// Given
 /// \code
 ///   class A { void func(); };
 ///   class B { void member(); };
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// \c cxxRecordDecl(hasMethod(hasName("func"))) matches the declaration of
-/// \c A but not \c B.
+/// The matcher \matcher{cxxRecordDecl(hasMethod(hasName("func")))}
+/// matches the declaration of \match{class A { void func(); }}
+/// but does not match \nomatch{class B { void member(); }}
 AST_MATCHER_P(CXXRecordDecl, hasMethod, internal::Matcher<CXXMethodDecl>,
               InnerMatcher) {
   BoundNodesTreeBuilder Result(*Builder);
@@ -3447,13 +4795,14 @@ AST_MATCHER_P(CXXRecordDecl, hasMethod, internal::Matcher<CXXMethodDecl>,
 
 /// Matches the generated class of lambda expressions.
 ///
-/// Given:
+/// Given
 /// \code
 ///   auto x = []{};
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c cxxRecordDecl(isLambda()) matches the implicit class declaration of
-/// \c decltype(x)
+/// The matcher \matcher{varDecl(hasType(cxxRecordDecl(isLambda())))}
+/// matches \match{auto x = []{}}.
 AST_MATCHER(CXXRecordDecl, isLambda) {
   return Node.isLambda();
 }
@@ -3461,33 +4810,53 @@ AST_MATCHER(CXXRecordDecl, isLambda) {
 /// Matches AST nodes that have child AST nodes that match the
 /// provided matcher.
 ///
-/// Example matches X, Y
-///   (matcher = cxxRecordDecl(has(cxxRecordDecl(hasName("X")))
+/// Given
 /// \code
 ///   class X {};  // Matches X, because X::X is a class of name X inside X.
 ///   class Y { class X {}; };
 ///   class Z { class Y { class X {}; }; };  // Does not match Z.
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(has(cxxRecordDecl(hasName("X"))))}
+/// matches \match{count=3$class X {}} three times for the definitions of \c X
+/// that contain the implicit class declarations of \c X ,
+/// and \match{count=2$class Y { class X {}; }} two times for the two different
+/// definitions of \c Y that contain \c X .
 ///
 /// ChildT must be an AST base type.
 ///
 /// Usable as: Any Matcher
 /// Note that has is direct matcher, so it also matches things like implicit
 /// casts and paren casts. If you are matching with expr then you should
-/// probably consider using ignoringParenImpCasts like:
-/// has(ignoringParenImpCasts(expr())).
+/// probably consider using ignoringParenImpCasts:
+///
+/// Given
+/// \code
+///   int x =0;
+///   double y = static_cast<double>(x);
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxStaticCastExpr(has(ignoringParenImpCasts(declRefExpr())))}.
+/// matches \match{static_cast<double>(x)}
 extern const internal::ArgumentAdaptingMatcherFunc<internal::HasMatcher> has;
 
 /// Matches AST nodes that have descendant AST nodes that match the
 /// provided matcher.
 ///
-/// Example matches X, Y, Z
-///     (matcher = cxxRecordDecl(hasDescendant(cxxRecordDecl(hasName("X")))))
+/// Given
 /// \code
 ///   class X {};  // Matches X, because X::X is a class of name X inside X.
 ///   class Y { class X {}; };
 ///   class Z { class Y { class X {}; }; };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasDescendant(cxxRecordDecl(hasName("X"))))}
+/// matches \match{count=3$class X {}} three times for the definitions of \c X
+/// that contain the implicit class declarations of \c X ,
+/// \match{count=2$class Y { class X {}; }} two times for the declaration of
+/// \c X they contain, and \match{class Z { class Y { class X {}; }; }}.
 ///
 /// DescendantT must be an AST base type.
 ///
@@ -3499,19 +4868,24 @@ extern const internal::ArgumentAdaptingMatcherFunc<
 /// Matches AST nodes that have child AST nodes that match the
 /// provided matcher.
 ///
-/// Example matches X, Y, Y::X, Z::Y, Z::Y::X
-///   (matcher = cxxRecordDecl(forEach(cxxRecordDecl(hasName("X")))
+/// Given
 /// \code
 ///   class X {};
 ///   class Y { class X {}; };  // Matches Y, because Y::X is a class of name X
 ///                             // inside Y.
 ///   class Z { class Y { class X {}; }; };  // Does not match Z.
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(forEach(cxxRecordDecl(hasName("X"))))}
+/// matches \match{type=typestr$class X},
+/// \match{type=typestr$class Y},
+/// \match{type=typestr$class Y::X},
+/// \match{type=typestr$class Z::Y::X} and \match{type=typestr$class Z::Y}
 ///
 /// ChildT must be an AST base type.
 ///
 /// As opposed to 'has', 'forEach' will cause a match for each result that
-/// matches instead of only on the first one.
+///   matches instead of only on the first one.
 ///
 /// Usable as: Any Matcher
 extern const internal::ArgumentAdaptingMatcherFunc<internal::ForEachMatcher>
@@ -3520,14 +4894,21 @@ extern const internal::ArgumentAdaptingMatcherFunc<internal::ForEachMatcher>
 /// Matches AST nodes that have descendant AST nodes that match the
 /// provided matcher.
 ///
-/// Example matches X, A, A::X, B, B::C, B::C::X
-///   (matcher = cxxRecordDecl(forEachDescendant(cxxRecordDecl(hasName("X")))))
+/// Given
 /// \code
 ///   class X {};
 ///   class A { class X {}; };  // Matches A, because A::X is a class of name
 ///                             // X inside A.
 ///   class B { class C { class X {}; }; };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(forEachDescendant(cxxRecordDecl(hasName("X"))))}
+/// matches \match{count=3$class X {}} three times, once for each of the
+/// declared classes \c X and their implicit class declaration,
+/// \match{class A { class X {}; }},
+/// \match{class B { class C { class X {}; }; }} and
+/// \match{class C { class X {}; }}.
 ///
 /// DescendantT must be an AST base type.
 ///
@@ -3535,13 +4916,63 @@ extern const internal::ArgumentAdaptingMatcherFunc<internal::ForEachMatcher>
 /// each result that matches instead of only on the first one.
 ///
 /// Note: Recursively combined ForEachDescendant can cause many matches:
-///   cxxRecordDecl(forEachDescendant(cxxRecordDecl(
-///     forEachDescendant(cxxRecordDecl())
-///   )))
-/// will match 10 times (plus injected class name matches) on:
 /// \code
-///   class A { class B { class C { class D { class E {}; }; }; }; };
+///   struct A {
+///     struct B {
+///       struct C {};
+///       struct D {};
+///     };
+///   };
+///
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(forEachDescendant(cxxRecordDecl(
+///     forEachDescendant(cxxRecordDecl().bind("inner"))
+///   ).bind("middle")))}
+/// will match 9 times:
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$B} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$B} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$C} in the middle and the definition of
+/// \match{sub=inner;type=name$B} as the innermost \c cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$C} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$B} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$B} in the middle and the definition of
+/// \match{sub=inner;type=name$D} as the innermost \c cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$B} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$D} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$C} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$C} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$A} with the definition of
+/// \match{sub=middle;type=name$D} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$D} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$B} with the definition of
+/// \match{sub=middle;type=name$C} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$C} as the innermost \c
+/// cxxRecordDecl.
+///
+/// It matches the definition of \match{type=name$B} with the definition of
+/// \match{sub=middle;type=name$D} in the middle and the implicit class
+/// declaration of \match{sub=inner;type=name$D} as the innermost \c
+/// cxxRecordDecl.
 ///
 /// Usable as: Any Matcher
 extern const internal::ArgumentAdaptingMatcherFunc<
@@ -3556,12 +4987,14 @@ extern const internal::ArgumentAdaptingMatcherFunc<
 /// \code
 ///   class A { class B {}; class C {}; };
 /// \endcode
-/// The matcher:
-/// \code
-///   cxxRecordDecl(hasName("::A"),
-///                 findAll(cxxRecordDecl(isDefinition()).bind("m")))
-/// \endcode
-/// will generate results for \c A, \c B and \c C.
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasName("::A"),
+///                 findAll(cxxRecordDecl(isDefinition()).bind("m")))}
+/// matches \match{count=3$class A { class B {}; class C {}; }} three times,
+/// with \matcher{type=sub$cxxRecordDecl(isDefinition()).bind("m")}
+/// matching \match{type=name;sub=m$A},
+/// \match{type=name;sub=m$B} and \match{type=name;sub=m$C}.
 ///
 /// Usable as: Any Matcher
 template <typename T>
@@ -3576,7 +5009,9 @@ internal::Matcher<T> findAll(const internal::Matcher<T> &Matcher) {
 /// \code
 /// void f() { for (;;) { int x = 42; if (true) { int x = 43; } } }
 /// \endcode
-/// \c compoundStmt(hasParent(ifStmt())) matches "{ int x = 43; }".
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{compoundStmt(hasParent(ifStmt()))}
+/// matches \match{{ int x = 43; }}
 ///
 /// Usable as: Any Matcher
 extern const internal::ArgumentAdaptingMatcherFunc<
@@ -3593,7 +5028,10 @@ extern const internal::ArgumentAdaptingMatcherFunc<
 /// void f() { if (true) { int x = 42; } }
 /// void g() { for (;;) { int x = 43; } }
 /// \endcode
-/// \c expr(integerLiteral(hasAncestor(ifStmt()))) matches \c 42, but not 43.
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{expr(integerLiteral(hasAncestor(ifStmt())))}
+/// matches \match{42}
+/// but does not match \nomatch{43}
 ///
 /// Usable as: Any Matcher
 extern const internal::ArgumentAdaptingMatcherFunc<
@@ -3604,17 +5042,20 @@ extern const internal::ArgumentAdaptingMatcherFunc<
 
 /// Matches if the provided matcher does not match.
 ///
-/// Example matches Y (matcher = cxxRecordDecl(unless(hasName("X"))))
+/// Given
 /// \code
-///   class X {};
-///   class Y {};
+///   int x;
+///   int y = 0;
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{varDecl(unless(hasInitializer(expr())))}
+/// matches \match{int x}, but not \nomatch{int y = 0}.
 ///
 /// Usable as: Any Matcher
 extern const internal::VariadicOperatorMatcherFunc<1, 1> unless;
 
 /// Matches a node if the declaration associated with that node
-/// matches the given matcher.
+///   matches the given matcher.
 ///
 /// The associated declaration is:
 /// - for type nodes, the declaration of the underlying type
@@ -3624,21 +5065,27 @@ extern const internal::VariadicOperatorMatcherFunc<1, 1> unless;
 /// - for CXXNewExpr, the declaration of the operator new
 /// - for ObjCIvarExpr, the declaration of the ivar
 ///
-/// For type nodes, hasDeclaration will generally match the declaration of the
-/// sugared type. Given
+/// Given
 /// \code
 ///   class X {};
 ///   typedef X Y;
 ///   Y y;
 /// \endcode
-/// in varDecl(hasType(hasDeclaration(decl()))) the decl will match the
-/// typedefDecl. A common use case is to match the underlying, desugared type.
+/// \compile_args{-std=c++}
+/// For type nodes, hasDeclaration will generally match the declaration of the
+/// sugared type, i.e., the matcher
+/// \matcher{varDecl(hasType(qualType(hasDeclaration(decl().bind("d")))))},
+/// matches \match{Y y}, with
+/// the matcher \matcher{type=sub$decl()} matching
+/// \match{sub=d$typedef X Y;}.
+/// A common use case is to match the underlying, desugared type.
 /// This can be achieved by using the hasUnqualifiedDesugaredType matcher:
-/// \code
-///   varDecl(hasType(hasUnqualifiedDesugaredType(
-///       recordType(hasDeclaration(decl())))))
-/// \endcode
-/// In this matcher, the decl will match the CXXRecordDecl of class X.
+/// \matcher{varDecl(hasType(hasUnqualifiedDesugaredType(
+///       recordType(hasDeclaration(decl().bind("d"))))))}
+/// matches \match{Y y}.
+/// In this matcher, the matcher \matcher{type=sub$decl()} will match the
+/// CXXRecordDecl
+/// \match{sub=d$class X {};}.
 ///
 /// Usable as: Matcher<AddrLabelExpr>, Matcher<CallExpr>,
 ///   Matcher<CXXConstructExpr>, Matcher<CXXNewExpr>, Matcher<DeclRefExpr>,
@@ -3665,9 +5112,10 @@ hasDeclaration(const internal::Matcher<Decl> &InnerMatcher) {
 ///   namespace N { template<class T> void f(T t); }
 ///   template <class T> void g() { using N::f; f(T()); }
 /// \endcode
-/// \c unresolvedLookupExpr(hasAnyDeclaration(
-///     namedDecl(hasUnderlyingDecl(hasName("::N::f")))))
-///   matches the use of \c f in \c g() .
+/// \compile_args{-fno-delayed-template-parsing;-std=c++11-or-later}
+/// The matcher \matcher{unresolvedLookupExpr(hasAnyDeclaration(
+///     namedDecl(hasUnderlyingDecl(hasName("::N::f")))))}
+///   matches \match{f} in \c g().
 AST_MATCHER_P(NamedDecl, hasUnderlyingDecl, internal::Matcher<NamedDecl>,
               InnerMatcher) {
   const NamedDecl *UnderlyingDecl = Node.getUnderlyingDecl();
@@ -3686,12 +5134,15 @@ AST_MATCHER_P(NamedDecl, hasUnderlyingDecl, internal::Matcher<NamedDecl>,
 ///   class X : public Y {};
 ///   void z(Y y, X x) { y.m(); (g()).m(); x.m(); }
 /// \endcode
-/// cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("Y")))))
-///   matches `y.m()` and `(g()).m()`.
-/// cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("X")))))
-///   matches `x.m()`.
-/// cxxMemberCallExpr(on(callExpr()))
-///   matches `(g()).m()`.
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("Y")))))}
+///   matches \match{y.m()} and \match{(g()).m()}.
+/// The matcher
+/// \matcher{cxxMemberCallExpr(on(hasType(cxxRecordDecl(hasName("X")))))}
+///   matches \match{x.m()}.
+/// The matcher \matcher{cxxMemberCallExpr(on(callExpr()))}
+///   matches \match{(g()).m()}.
 ///
 /// FIXME: Overload to allow directly matching types?
 AST_MATCHER_P(CXXMemberCallExpr, on, internal::Matcher<Expr>,
@@ -3702,17 +5153,17 @@ AST_MATCHER_P(CXXMemberCallExpr, on, internal::Matcher<Expr>,
           InnerMatcher.matches(*ExprNode, Finder, Builder));
 }
 
-
 /// Matches on the receiver of an ObjectiveC Message expression.
 ///
-/// Example
-/// matcher = objCMessageExpr(hasReceiverType(asString("UIWebView *")));
-/// matches the [webView ...] message invocation.
 /// \code
 ///   NSString *webViewJavaScript = ...
 ///   UIWebView *webView = ...
 ///   [webView stringByEvaluatingJavaScriptFromString:webViewJavascript];
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objCMessageExpr(hasReceiverType(asString("UIWebView
+/// *")))} matches
+/// \match{[webViewstringByEvaluatingJavaScriptFromString:webViewJavascript];}
 AST_MATCHER_P(ObjCMessageExpr, hasReceiverType, internal::Matcher<QualType>,
               InnerMatcher) {
   const QualType TypeDecl = Node.getReceiverType();
@@ -3721,66 +5172,63 @@ AST_MATCHER_P(ObjCMessageExpr, hasReceiverType, internal::Matcher<QualType>,
 
 /// Returns true when the Objective-C method declaration is a class method.
 ///
-/// Example
-/// matcher = objcMethodDecl(isClassMethod())
-/// matches
+/// Given
 /// \code
 /// @interface I + (void)foo; @end
-/// \endcode
-/// but not
-/// \code
 /// @interface I - (void)bar; @end
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objcMethodDecl(isClassMethod())}
+/// matches \match{@interface I + (void)foo; @end}
+/// but does not match \nomatch{interface I + (void)foo; @end}
 AST_MATCHER(ObjCMethodDecl, isClassMethod) {
   return Node.isClassMethod();
 }
 
 /// Returns true when the Objective-C method declaration is an instance method.
 ///
-/// Example
-/// matcher = objcMethodDecl(isInstanceMethod())
-/// matches
+/// Given
 /// \code
 /// @interface I - (void)bar; @end
-/// \endcode
-/// but not
-/// \code
 /// @interface I + (void)foo; @end
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objcMethodDecl(isInstanceMethod())}
+/// matches \match{@interface I - (void)bar; @end}
+/// but does not match \nomatch{@interface I - (void)foo; @end}
+/// \compile_args{-ObjC}
 AST_MATCHER(ObjCMethodDecl, isInstanceMethod) {
   return Node.isInstanceMethod();
 }
 
 /// Returns true when the Objective-C message is sent to a class.
 ///
-/// Example
-/// matcher = objcMessageExpr(isClassMessage())
-/// matches
+/// Given
 /// \code
 ///   [NSString stringWithFormat:@"format"];
-/// \endcode
-/// but not
-/// \code
 ///   NSString *x = @"hello";
 ///   [x containsString:@"h"];
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objcMessageExpr(isClassMessage())}
+/// matches \match{[NSString stringWithFormat:@"format"];}
+/// but does not match \nomatch{[[x containsString:@"h"]}
 AST_MATCHER(ObjCMessageExpr, isClassMessage) {
   return Node.isClassMessage();
 }
 
 /// Returns true when the Objective-C message is sent to an instance.
 ///
-/// Example
-/// matcher = objcMessageExpr(isInstanceMessage())
-/// matches
+/// Given
 /// \code
 ///   NSString *x = @"hello";
 ///   [x containsString:@"h"];
-/// \endcode
-/// but not
-/// \code
 ///   [NSString stringWithFormat:@"format"];
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objcMessageExpr(isInstanceMessage())}
+/// matches \match{[x containsString:@"h"];}
+/// but does not match \nomatch{[NSString stringWithFormat:@"format"];}
 AST_MATCHER(ObjCMessageExpr, isInstanceMessage) {
   return Node.isInstanceMessage();
 }
@@ -3788,13 +5236,15 @@ AST_MATCHER(ObjCMessageExpr, isInstanceMessage) {
 /// Matches if the Objective-C message is sent to an instance,
 /// and the inner matcher matches on that instance.
 ///
-/// For example the method call in
+/// Given
 /// \code
 ///   NSString *x = @"hello";
 ///   [x containsString:@"h"];
 /// \endcode
-/// is matched by
-/// objcMessageExpr(hasReceiver(declRefExpr(to(varDecl(hasName("x"))))))
+/// \compile_args{-ObjC}
+/// The matcher
+/// \matcher{objcMessageExpr(hasReceiver(declRefExpr(to(varDecl(hasName("x"))))))}
+/// matches \match{[x containsString:@"h"];}
 AST_MATCHER_P(ObjCMessageExpr, hasReceiver, internal::Matcher<Expr>,
               InnerMatcher) {
   const Expr *ReceiverNode = Node.getInstanceReceiver();
@@ -3805,12 +5255,14 @@ AST_MATCHER_P(ObjCMessageExpr, hasReceiver, internal::Matcher<Expr>,
 
 /// Matches when BaseName == Selector.getAsString()
 ///
-///  matcher = objCMessageExpr(hasSelector("loadHTMLString:baseURL:"));
-///  matches the outer message expr in the code below, but NOT the message
-///  invocation for self.bodyView.
 /// \code
 ///     [self.bodyView loadHTMLString:html baseURL:NULL];
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher
+/// \matcher{objCMessageExpr(hasSelector("loadHTMLString:baseURL:"));} matches
+/// the outer message expr in the code below, but NOT the message invocation
+/// for self.bodyView.
 AST_MATCHER_P(ObjCMessageExpr, hasSelector, std::string, BaseName) {
   Selector Sel = Node.getSelector();
   return BaseName == Sel.getAsString();
@@ -3819,12 +5271,13 @@ AST_MATCHER_P(ObjCMessageExpr, hasSelector, std::string, BaseName) {
 /// Matches when at least one of the supplied string equals to the
 /// Selector.getAsString()
 ///
-///  matcher = objCMessageExpr(hasSelector("methodA:", "methodB:"));
-///  matches both of the expressions below:
 /// \code
 ///     [myObj methodA:argA];
 ///     [myObj methodB:argB];
 /// \endcode
+/// \compile_args{-ObjC}
+///  The matcher \matcher{objCMessageExpr(hasSelector("methodA:", "methodB:"));}
+///  matches \match{[myObj methodA:argA];} and \match{[myObj methodB:argB];}
 extern const internal::VariadicFunction<internal::Matcher<ObjCMessageExpr>,
                                         StringRef,
                                         internal::hasAnySelectorFunc>
@@ -3832,12 +5285,17 @@ extern const internal::VariadicFunction<internal::Matcher<ObjCMessageExpr>,
 
 /// Matches ObjC selectors whose name contains
 /// a substring matched by the given RegExp.
-///  matcher = objCMessageExpr(matchesSelector("loadHTMLString\:baseURL?"));
-///  matches the outer message expr in the code below, but NOT the message
-///  invocation for self.bodyView.
+///
+/// Given
 /// \code
 ///     [self.bodyView loadHTMLString:html baseURL:NULL];
 /// \endcode
+/// \compile_args{-ObjC}
+///
+/// The matcher
+/// \matcher{objCMessageExpr(matchesSelector("loadHTMLString\:baseURL?"))}
+/// matches the outer message expr in the code below, but NOT the message
+/// invocation for self.bodyView.
 AST_MATCHER_REGEX(ObjCMessageExpr, matchesSelector, RegExp) {
   std::string SelectorString = Node.getSelector().getAsString();
   return RegExp->match(SelectorString);
@@ -3853,21 +5311,22 @@ AST_MATCHER(ObjCMessageExpr, hasNullSelector) {
 
 /// Matches when the selector is a Unary Selector
 ///
-///  matcher = objCMessageExpr(matchesSelector(hasUnarySelector());
-///  matches self.bodyView in the code below, but NOT the outer message
-///  invocation of "loadHTMLString:baseURL:".
+/// Given
 /// \code
 ///     [self.bodyView loadHTMLString:html baseURL:NULL];
 /// \endcode
+/// \compile_args{-ObjC}
+///
+///  The matcher \matcher{objCMessageExpr(matchesSelector(hasUnarySelector());}
+///  matches \match{self.bodyView}, but does not match the outer message
+///  invocation of "loadHTMLString:baseURL:".
 AST_MATCHER(ObjCMessageExpr, hasUnarySelector) {
   return Node.getSelector().isUnarySelector();
 }
 
 /// Matches when the selector is a keyword selector
 ///
-/// objCMessageExpr(hasKeywordSelector()) matches the generated setFrame
-/// message expression in
-///
+/// Given
 /// \code
 ///   UIWebView *webView = ...;
 ///   CGRect bodyFrame = webView.frame;
@@ -3875,21 +5334,25 @@ AST_MATCHER(ObjCMessageExpr, hasUnarySelector) {
 ///   webView.frame = bodyFrame;
 ///   //     ^---- matches here
 /// \endcode
+/// \compile_args{-ObjC}
+///
+/// The matcher \matcher{objCMessageExpr(hasKeywordSelector())} matches the
+/// generated setFrame message expression in
 AST_MATCHER(ObjCMessageExpr, hasKeywordSelector) {
   return Node.getSelector().isKeywordSelector();
 }
 
 /// Matches when the selector has the specified number of arguments
 ///
-///  matcher = objCMessageExpr(numSelectorArgs(0));
-///  matches self.bodyView in the code below
-///
-///  matcher = objCMessageExpr(numSelectorArgs(2));
-///  matches the invocation of "loadHTMLString:baseURL:" but not that
-///  of self.bodyView
 /// \code
 ///     [self.bodyView loadHTMLString:html baseURL:NULL];
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objCMessageExpr(numSelectorArgs(0))}
+/// matches \match{self.bodyView}.
+/// The matcher \matcher{objCMessageExpr(numSelectorArgs(2))}
+/// matches the invocation of \match{loadHTMLString:baseURL:}
+/// but does not match \nomatch{self.bodyView}
 AST_MATCHER_P(ObjCMessageExpr, numSelectorArgs, unsigned, N) {
   return Node.getSelector().getNumArgs() == N;
 }
@@ -3901,13 +5364,21 @@ AST_MATCHER_P(ObjCMessageExpr, numSelectorArgs, unsigned, N) {
 ///   class Y { void x() { this->x(); x(); Y y; y.x(); } };
 ///   void f() { f(); }
 /// \endcode
-/// callExpr(callee(expr()))
-///   matches this->x(), x(), y.x(), f()
-/// with callee(...)
-///   matching this->x, x, y.x, f respectively
+/// \compile_args{-std=c++}
+/// The matcher \matcher{callExpr(callee(expr().bind("callee")))}
+/// matches \match{this->x()}, \match{x()}, \match{y.x()}, \match{f()}
+/// with \matcher{type=sub$expr()} inside of \c callee
+/// matching \match{sub=callee$this->x}, \match{sub=callee$x},
+/// \match{sub=callee$y.x}, \match{sub=callee$f} respectively
 ///
 /// Given
 /// \code
+///   struct Dummy {};
+///   // makes sure there is a callee, otherwise there would be no callee,
+///   // just a builtin operator
+///   Dummy operator+(Dummy, Dummy);
+///   // not defining a '*' operator
+///
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
 ///       return (0 + ... + args);
@@ -3918,10 +5389,14 @@ AST_MATCHER_P(ObjCMessageExpr, numSelectorArgs, unsigned, N) {
 ///       return (args * ... * 1);
 ///   }
 /// \endcode
-/// cxxFoldExpr(callee(expr()))
-///   matches (args * ... * 1)
-/// with callee(...)
-///   matching *
+/// \compile_args{-std=c++17-or-later}
+/// The matcher \matcher{cxxFoldExpr(callee(expr().bind("op")))}
+/// matches \match{(0 + ... + args)}
+/// with \matcher{type=sub$callee(...)} matching \match{sub=op$*},
+/// but does not match \nomatch{(args * ... * 1)}.
+/// A \c CXXFoldExpr only has an \c UnresolvedLookupExpr as a callee.
+/// When there are no define operators that could be used instead of builtin
+/// ones, then there will be no \c callee .
 ///
 /// Note: Callee cannot take the more general internal::Matcher<Expr>
 /// because this introduces ambiguous overloads with calls to Callee taking a
@@ -3940,16 +5415,16 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(callee,
 /// given matcher; or 2) if the Obj-C message expression's callee's method
 /// declaration matches the given matcher.
 ///
-/// Example matches y.x() (matcher = callExpr(callee(
-///                                    cxxMethodDecl(hasName("x")))))
+/// Example 1
 /// \code
 ///   class Y { public: void x(); };
 ///   void z() { Y y; y.x(); }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{callExpr(callee(cxxMethodDecl(hasName("x"))))}
+/// matches \match{y.x()}
 ///
-/// Example 2. Matches [I foo] with
-/// objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))
-///
+/// Example 2
 /// \code
 ///   @interface I: NSObject
 ///   +(void)foo;
@@ -3957,6 +5432,10 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(callee,
 ///   ...
 ///   [I foo]
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher
+/// \matcher{objcMessageExpr(callee(objcMethodDecl(hasName("foo"))))}
+/// matches \match{[I foo]}
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     callee, AST_POLYMORPHIC_SUPPORTED_TYPES(ObjCMessageExpr, CallExpr),
     internal::Matcher<Decl>, InnerMatcher, 1) {
@@ -3976,12 +5455,7 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 /// Matches if the expression's or declaration's type matches a type
 /// matcher.
 ///
-/// Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-///             and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-///             and U (matcher = typedefDecl(hasType(asString("int")))
-///             and friend class X (matcher = friendDecl(hasType("X"))
-///             and public virtual X (matcher = cxxBaseSpecifier(hasType(
-///                                               asString("class X")))
+/// Exmaple
 /// \code
 ///  class X {};
 ///  void y(X &x) { x; X z; }
@@ -3989,6 +5463,19 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 ///  class Y { friend class X; };
 ///  class Z : public virtual X {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{expr(hasType(cxxRecordDecl(hasName("X"))))}
+/// matches \match{x} and \match{z}.
+/// The matcher \matcher{varDecl(hasType(cxxRecordDecl(hasName("X"))))}
+/// matches \match{X z}
+/// The matcher \matcher{typedefDecl(hasType(asString("int")))}
+/// matches \match{typedef int U}
+/// The matcher \matcher{friendDecl(hasType(asString("class X")))}
+/// matches \match{friend class X}
+/// The matcher \matcher{cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+/// asString("X"))).bind("b")))} matches \match{class Z : public virtual X {}},
+/// with \matcher{type=sub$cxxBaseSpecifier(...)}
+/// matching \match{sub=b$public virtual X}.
 AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
     hasType,
     AST_POLYMORPHIC_SUPPORTED_TYPES(Expr, FriendDecl, TypedefNameDecl,
@@ -4009,24 +5496,34 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 /// X, while varDecl(hasType(cxxRecordDecl(hasName("X")))) matches the
 /// declaration of x.
 ///
-/// Example matches x (matcher = expr(hasType(cxxRecordDecl(hasName("X")))))
-///             and z (matcher = varDecl(hasType(cxxRecordDecl(hasName("X")))))
-///             and friend class X (matcher = friendDecl(hasType("X"))
-///             and public virtual X (matcher = cxxBaseSpecifier(hasType(
-///                                               cxxRecordDecl(hasName("X"))))
 /// \code
 ///  class X {};
 ///  void y(X &x) { x; X z; }
 ///  class Y { friend class X; };
 ///  class Z : public virtual X {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{expr(hasType(cxxRecordDecl(hasName("X"))))}
+/// matches \match{x} and \match{z}.
+/// The matcher \matcher{varDecl(hasType(cxxRecordDecl(hasName("X"))))}
+/// matches \match{X z}.
+/// The matcher \matcher{friendDecl(hasType(asString("class X")))}
+/// matches \match{friend class X}.
+/// The matcher \matcher{cxxRecordDecl(hasAnyBase(cxxBaseSpecifier(hasType(
+/// asString("X"))).bind("b")))} matches
+/// \match{class Z : public virtual X {}},
+/// with \matcher{type=sub$cxxBaseSpecifier(...)}
+/// matching \match{sub=b$public virtual X}.
 ///
-/// Example matches class Derived
-/// (matcher = cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base"))))))
+/// Given
 /// \code
 /// class Base {};
 /// class Derived : Base {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasAnyBase(hasType(cxxRecordDecl(hasName("Base")))))}
+/// matches \match{class Derived : Base {}}.
 ///
 /// Usable as: Matcher<Expr>, Matcher<FriendDecl>, Matcher<ValueDecl>,
 /// Matcher<CXXBaseSpecifier>
@@ -4043,25 +5540,32 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(
 
 /// Matches if the type location of a node matches the inner matcher.
 ///
-/// Examples:
+/// Given
 /// \code
 ///   int x;
 /// \endcode
-/// declaratorDecl(hasTypeLoc(loc(asString("int"))))
-///   matches int x
+/// The matcher \matcher{declaratorDecl(hasTypeLoc(loc(asString("int"))))}
+/// matches \match{int x}.
 ///
+/// Given
 /// \code
-/// auto x = int(3);
+/// struct point { point(double, double); };
+/// point p = point(1.0, -1.0);
 /// \endcode
-/// cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("int"))))
-///   matches int(3)
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxTemporaryObjectExpr(hasTypeLoc(loc(asString("point"))))}
+/// matches \match{point(1.0, -1.0)}.
 ///
+/// Given
 /// \code
 /// struct Foo { Foo(int, int); };
-/// auto x = Foo(1, 2);
+/// Foo x = Foo(1, 2);
 /// \endcode
-/// cxxFunctionalCastExpr(hasTypeLoc(loc(asString("struct Foo"))))
-///   matches Foo(1, 2)
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxTemporaryObjectExpr(hasTypeLoc(
+///                           loc(asString("Foo"))))}
+/// matches \match{Foo(1, 2)}.
 ///
 /// Usable as: Matcher<BlockDecl>, Matcher<CXXBaseSpecifier>,
 ///   Matcher<CXXCtorInitializer>, Matcher<CXXFunctionalCastExpr>,
@@ -4094,22 +5598,25 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   class Y { public: void x(); };
 ///   void z() { Y* y; y->x(); }
 /// \endcode
-/// cxxMemberCallExpr(on(hasType(asString("class Y *"))))
-///   matches y->x()
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMemberCallExpr(on(hasType(asString("Y *"))))}
+/// matches \match{y->x()}
 AST_MATCHER_P(QualType, asString, std::string, Name) {
   return Name == Node.getAsString();
 }
 
 /// Matches if the matched type is a pointer type and the pointee type
-/// matches the specified matcher.
+///   matches the specified matcher.
 ///
-/// Example matches y->x()
-///   (matcher = cxxMemberCallExpr(on(hasType(pointsTo
-///      cxxRecordDecl(hasName("Y")))))))
+/// Given
 /// \code
 ///   class Y { public: void x(); };
 ///   void z() { Y *y; y->x(); }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMemberCallExpr(on(hasType(pointsTo(
+///      qualType()))))}
+/// matches \match{y->x()}
 AST_MATCHER_P(
     QualType, pointsTo, internal::Matcher<QualType>,
     InnerMatcher) {
@@ -4117,7 +5624,19 @@ AST_MATCHER_P(
           InnerMatcher.matches(Node->getPointeeType(), Finder, Builder));
 }
 
+/// Matches if the matched type is a pointer type and the pointee type
+///   matches the specified matcher.
 /// Overloaded to match the pointee type's declaration.
+///
+/// Given
+/// \code
+///   class Y { public: void x(); };
+///   void z() { Y *y; y->x(); }
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMemberCallExpr(on(hasType(pointsTo(
+///      cxxRecordDecl(hasName("Y"))))))}
+/// matches \match{y->x()}
 AST_MATCHER_P_OVERLOAD(QualType, pointsTo, internal::Matcher<Decl>,
                        InnerMatcher, 1) {
   return pointsTo(qualType(hasDeclaration(InnerMatcher)))
@@ -4131,9 +5650,12 @@ AST_MATCHER_P_OVERLOAD(QualType, pointsTo, internal::Matcher<Decl>,
 /// \code
 ///   class A {};
 ///   using B = A;
+///   B b;
 /// \endcode
-/// The matcher type(hasUnqualifiedDesugaredType(recordType())) matches
-/// both B and A.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{varDecl(hasType(hasUnqualifiedDesugaredType(recordType())))}
+/// matches \match{B b}.
 AST_MATCHER_P(Type, hasUnqualifiedDesugaredType, internal::Matcher<Type>,
               InnerMatcher) {
   return InnerMatcher.matches(*Node.getUnqualifiedDesugaredType(), Finder,
@@ -4143,8 +5665,7 @@ AST_MATCHER_P(Type, hasUnqualifiedDesugaredType, internal::Matcher<Type>,
 /// Matches if the matched type is a reference type and the referenced
 /// type matches the specified matcher.
 ///
-/// Example matches X &x and const X &y
-///     (matcher = varDecl(hasType(references(cxxRecordDecl(hasName("X"))))))
+/// Given
 /// \code
 ///   class X {
 ///     void a(X b) {
@@ -4153,24 +5674,29 @@ AST_MATCHER_P(Type, hasUnqualifiedDesugaredType, internal::Matcher<Type>,
 ///     }
 ///   };
 /// \endcode
-AST_MATCHER_P(QualType, references, internal::Matcher<QualType>,
-              InnerMatcher) {
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{varDecl(hasType(references(qualType())))} matches
+/// \match{X &x = b} and \match{const X &y = b}.
+AST_MATCHER_P(QualType, references, internal::Matcher<QualType>, InnerMatcher) {
   return (!Node.isNull() && Node->isReferenceType() &&
           InnerMatcher.matches(Node->getPointeeType(), Finder, Builder));
 }
 
 /// Matches QualTypes whose canonical type matches InnerMatcher.
 ///
-/// Given:
+/// Given
 /// \code
 ///   typedef int &int_ref;
 ///   int a;
 ///   int_ref b = a;
 /// \endcode
-///
-/// \c varDecl(hasType(qualType(referenceType()))))) will not match the
-/// declaration of b but \c
-/// varDecl(hasType(qualType(hasCanonicalType(referenceType())))))) does.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{varDecl(hasType(qualType(referenceType())))}
+/// does not match \nomatch{int_ref b = a},
+/// but the matcher
+/// \matcher{varDecl(hasType(qualType(hasCanonicalType(referenceType()))))}
+/// does match \match{int_ref b = a}.
 AST_MATCHER_P(QualType, hasCanonicalType, internal::Matcher<QualType>,
               InnerMatcher) {
   if (Node.isNull())
@@ -4178,7 +5704,23 @@ AST_MATCHER_P(QualType, hasCanonicalType, internal::Matcher<QualType>,
   return InnerMatcher.matches(Node.getCanonicalType(), Finder, Builder);
 }
 
+/// Matches if the matched type is a reference type and the referenced
+/// type matches the specified matcher.
 /// Overloaded to match the referenced type's declaration.
+///
+/// Given
+/// \code
+///   class X {
+///     void a(X b) {
+///       X &x = b;
+///       const X &y = b;
+///     }
+///   };
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{varDecl(hasType(references(cxxRecordDecl(hasName("X")))))} matches
+/// \match{X &x = b} and \match{const X &y = b}.
 AST_MATCHER_P_OVERLOAD(QualType, references, internal::Matcher<Decl>,
                        InnerMatcher, 1) {
   return references(qualType(hasDeclaration(InnerMatcher)))
@@ -4192,14 +5734,17 @@ AST_MATCHER_P_OVERLOAD(QualType, references, internal::Matcher<Decl>,
 /// \code
 ///   class Y { public: void m(); };
 ///   Y g();
-///   class X : public Y { void g(); };
+///   class X : public Y { public: void g(); };
 ///   void z(Y y, X x) { y.m(); x.m(); x.g(); (g()).m(); }
 /// \endcode
-/// cxxMemberCallExpr(onImplicitObjectArgument(hasType(
-///     cxxRecordDecl(hasName("Y")))))
-///   matches `y.m()`, `x.m()` and (g()).m(), but not `x.g()`.
-/// cxxMemberCallExpr(on(callExpr()))
-///   does not match `(g()).m()`, because the parens are not ignored.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMemberCallExpr(onImplicitObjectArgument(hasType(
+///     cxxRecordDecl(hasName("Y")))))}
+/// matches \match{y.m()}, \match{x.m()} and \match{(g()).m()}
+/// but does not match \nomatch{x.g()}.
+/// The matcher \matcher{cxxMemberCallExpr(on(callExpr()))}
+/// matches \match{(g()).m()}, because the parens are ignored.
+/// FIXME: should they be ignored? (ignored bc of `on`)
 ///
 /// FIXME: Overload to allow directly matching types?
 AST_MATCHER_P(CXXMemberCallExpr, onImplicitObjectArgument,
@@ -4210,21 +5755,20 @@ AST_MATCHER_P(CXXMemberCallExpr, onImplicitObjectArgument,
 }
 
 /// Matches if the type of the expression's implicit object argument either
-/// matches the InnerMatcher, or is a pointer to a type that matches the
+///   matches the InnerMatcher, or is a pointer to a type that matches the
 /// InnerMatcher.
 ///
 /// Given
 /// \code
-///   class Y { public: void m(); };
-///   class X : public Y { void g(); };
-///   void z() { Y y; y.m(); Y *p; p->m(); X x; x.m(); x.g(); }
+///   class Y { public: void m() const; };
+///   class X : public Y { public: void g(); };
+///   void z() { const Y y; y.m(); const Y *p; p->m(); X x; x.m(); x.g(); }
 /// \endcode
-/// cxxMemberCallExpr(thisPointerType(hasDeclaration(
-///     cxxRecordDecl(hasName("Y")))))
-///   matches `y.m()`, `p->m()` and `x.m()`.
-/// cxxMemberCallExpr(thisPointerType(hasDeclaration(
-///     cxxRecordDecl(hasName("X")))))
-///   matches `x.g()`.
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxMemberCallExpr(thisPointerType(isConstQualified()))}
+/// matches \match{y.m()}, \match{x.m()} and \match{p->m()},
+/// but not \nomatch{x.g()}.
 AST_MATCHER_P_OVERLOAD(CXXMemberCallExpr, thisPointerType,
                        internal::Matcher<QualType>, InnerMatcher, 0) {
   return onImplicitObjectArgument(
@@ -4233,6 +5777,20 @@ AST_MATCHER_P_OVERLOAD(CXXMemberCallExpr, thisPointerType,
 }
 
 /// Overloaded to match the type's declaration.
+///
+/// Given
+/// \code
+///   class Y { public: void m(); };
+///   class X : public Y { public: void g(); };
+///   void z() { Y y; y.m(); Y *p; p->m(); X x; x.m(); x.g(); }
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMemberCallExpr(thisPointerType(
+///     cxxRecordDecl(hasName("Y"))))}
+///   matches \match{y.m()}, \match{p->m()} and \match{x.m()}.
+/// The matcher \matcher{cxxMemberCallExpr(thisPointerType(
+///     cxxRecordDecl(hasName("X"))))}
+///   matches \match{x.g()}.
 AST_MATCHER_P_OVERLOAD(CXXMemberCallExpr, thisPointerType,
                        internal::Matcher<Decl>, InnerMatcher, 1) {
   return onImplicitObjectArgument(
@@ -4243,14 +5801,17 @@ AST_MATCHER_P_OVERLOAD(CXXMemberCallExpr, thisPointerType,
 /// Matches a DeclRefExpr that refers to a declaration that matches the
 /// specified matcher.
 ///
-/// Example matches x in if(x)
-///     (matcher = declRefExpr(to(varDecl(hasName("x")))))
+/// Given
 /// \code
-///   bool x;
-///   if (x) {}
+///   void foo() {
+///     bool x;
+///     if (x) {}
+///   }
 /// \endcode
-AST_MATCHER_P(DeclRefExpr, to, internal::Matcher<Decl>,
-              InnerMatcher) {
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{declRefExpr(to(varDecl(hasName("x"))))}
+/// matches \match{x} inside the condition of the if-stmt.
+AST_MATCHER_P(DeclRefExpr, to, internal::Matcher<Decl>, InnerMatcher) {
   const Decl *DeclNode = Node.getDecl();
   return (DeclNode != nullptr &&
           InnerMatcher.matches(*DeclNode, Finder, Builder));
@@ -4259,22 +5820,24 @@ AST_MATCHER_P(DeclRefExpr, to, internal::Matcher<Decl>,
 /// Matches if a node refers to a declaration through a specific
 /// using shadow declaration.
 ///
-/// Examples:
+/// Given
 /// \code
 ///   namespace a { int f(); }
 ///   using a::f;
 ///   int x = f();
 /// \endcode
-/// declRefExpr(throughUsingDecl(anything()))
-///   matches \c f
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{declRefExpr(throughUsingDecl(anything()))}
+/// matches \match{f}
 ///
 /// \code
 ///   namespace a { class X{}; }
 ///   using a::X;
 ///   X x;
 /// \endcode
-/// typeLoc(loc(usingType(throughUsingDecl(anything()))))
-///   matches \c X
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{typeLoc(loc(usingType(throughUsingDecl(anything()))))}
+/// matches \match{X}
 ///
 /// Usable as: Matcher<DeclRefExpr>, Matcher<UsingType>
 AST_POLYMORPHIC_MATCHER_P(throughUsingDecl,
@@ -4299,9 +5862,11 @@ AST_POLYMORPHIC_MATCHER_P(throughUsingDecl,
 ///     bar(t);
 ///   }
 /// \endcode
-/// unresolvedLookupExpr(hasAnyDeclaration(
-///     functionTemplateDecl(hasName("foo"))))
-///   matches \c foo in \c foo(t); but not \c bar in \c bar(t);
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unresolvedLookupExpr(hasAnyDeclaration(
+///     functionTemplateDecl(hasName("foo"))))}
+/// matches \match{foo} in \c foo(t);
+/// but does not match \nomatch{bar} in \c bar(t);
 AST_MATCHER_P(OverloadExpr, hasAnyDeclaration, internal::Matcher<Decl>,
               InnerMatcher) {
   return matchesFirstInPointerRange(InnerMatcher, Node.decls_begin(),
@@ -4313,11 +5878,14 @@ AST_MATCHER_P(OverloadExpr, hasAnyDeclaration, internal::Matcher<Decl>,
 ///
 /// Given
 /// \code
-///   int a, b;
-///   int c;
+///   void foo() {
+///     int a, b;
+///     int c;
+///   }
 /// \endcode
-/// declStmt(hasSingleDecl(anything()))
-///   matches 'int c;' but not 'int a, b;'.
+/// The matcher \matcher{declStmt(hasSingleDecl(anything()))}
+/// matches \match{int c;}
+/// but does not match \nomatch{int a, b;}
 AST_MATCHER_P(DeclStmt, hasSingleDecl, internal::Matcher<Decl>, InnerMatcher) {
   if (Node.isSingleDecl()) {
     const Decl *FoundDecl = Node.getSingleDecl();
@@ -4329,11 +5897,15 @@ AST_MATCHER_P(DeclStmt, hasSingleDecl, internal::Matcher<Decl>, InnerMatcher) {
 /// Matches a variable declaration that has an initializer expression
 /// that matches the given matcher.
 ///
-/// Example matches x (matcher = varDecl(hasInitializer(callExpr())))
+/// Given
 /// \code
-///   bool y() { return true; }
-///   bool x = y();
+///   int y() { return 0; }
+///   void foo() {
+///     int x = y();
+///   }
 /// \endcode
+/// The matcher \matcher{varDecl(hasInitializer(callExpr()))}
+/// matches \match{int x = y()}
 AST_MATCHER_P(
     VarDecl, hasInitializer, internal::Matcher<Expr>,
     InnerMatcher) {
@@ -4345,10 +5917,13 @@ AST_MATCHER_P(
 /// Matches a variable serving as the implicit variable for a lambda init-
 /// capture.
 ///
-/// Example matches x (matcher = varDecl(isInitCapture()))
+/// Given
 /// \code
-/// auto f = [x=3]() { return x; };
+/// auto f = [x = 3]() { return x; };
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{varDecl(isInitCapture())}
+/// matches \match{x = 3}.
 AST_MATCHER(VarDecl, isInitCapture) { return Node.isInitCapture(); }
 
 /// Matches each lambda capture in a lambda expression.
@@ -4356,14 +5931,19 @@ AST_MATCHER(VarDecl, isInitCapture) { return Node.isInitCapture(); }
 /// Given
 /// \code
 ///   int main() {
-///     int x, y;
+///     int x;
+///     int y;
 ///     float z;
 ///     auto f = [=]() { return x + y + z; };
 ///   }
 /// \endcode
-/// lambdaExpr(forEachLambdaCapture(
-///     lambdaCapture(capturesVar(varDecl(hasType(isInteger()))))))
-/// will trigger two matches, binding for 'x' and 'y' respectively.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{lambdaExpr(forEachLambdaCapture(
+///     lambdaCapture(capturesVar(
+///     varDecl(hasType(isInteger())).bind("captured")))))}
+/// matches \match{count=2$[=]() { return x + y + z; }} two times,
+/// with \matcher{type=sub$varDecl(hasType(isInteger()))} matching
+/// \match{sub=captured$int x} and \match{sub=captured$int y}.
 AST_MATCHER_P(LambdaExpr, forEachLambdaCapture,
               internal::Matcher<LambdaCapture>, InnerMatcher) {
   BoundNodesTreeBuilder Result;
@@ -4383,7 +5963,7 @@ AST_MATCHER_P(LambdaExpr, forEachLambdaCapture,
 
 /// \brief Matches a static variable with local scope.
 ///
-/// Example matches y (matcher = varDecl(isStaticLocal()))
+/// Given
 /// \code
 /// void f() {
 ///   int x;
@@ -4391,6 +5971,8 @@ AST_MATCHER_P(LambdaExpr, forEachLambdaCapture,
 /// }
 /// static int z;
 /// \endcode
+/// The matcher \matcher{varDecl(isStaticLocal())}
+/// matches \match{static int y}.
 AST_MATCHER(VarDecl, isStaticLocal) {
   return Node.isStaticLocal();
 }
@@ -4398,7 +5980,7 @@ AST_MATCHER(VarDecl, isStaticLocal) {
 /// Matches a variable declaration that has function scope and is a
 /// non-static local variable.
 ///
-/// Example matches x (matcher = varDecl(hasLocalStorage())
+/// Given
 /// \code
 /// void f() {
 ///   int x;
@@ -4406,13 +5988,15 @@ AST_MATCHER(VarDecl, isStaticLocal) {
 /// }
 /// int z;
 /// \endcode
+/// The matcher \matcher{varDecl(hasLocalStorage())}
+/// matches \match{int x}.
 AST_MATCHER(VarDecl, hasLocalStorage) {
   return Node.hasLocalStorage();
 }
 
 /// Matches a variable declaration that does not have local storage.
 ///
-/// Example matches y and z (matcher = varDecl(hasGlobalStorage())
+/// Given
 /// \code
 /// void f() {
 ///   int x;
@@ -4420,14 +6004,15 @@ AST_MATCHER(VarDecl, hasLocalStorage) {
 /// }
 /// int z;
 /// \endcode
+/// The matcher \matcher{varDecl(hasGlobalStorage())}
+/// matches \match{static int y} and \match{int z}.
 AST_MATCHER(VarDecl, hasGlobalStorage) {
   return Node.hasGlobalStorage();
 }
 
 /// Matches a variable declaration that has automatic storage duration.
 ///
-/// Example matches x, but not y, z, or a.
-/// (matcher = varDecl(hasAutomaticStorageDuration())
+/// Given
 /// \code
 /// void f() {
 ///   int x;
@@ -4436,6 +6021,11 @@ AST_MATCHER(VarDecl, hasGlobalStorage) {
 /// }
 /// int a;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{varDecl(hasAutomaticStorageDuration())}
+/// matches \match{int x}
+/// but does not match \nomatch{static int y}, \nomatch{thread_local int z} or
+/// \nomatch{int a}
 AST_MATCHER(VarDecl, hasAutomaticStorageDuration) {
   return Node.getStorageDuration() == SD_Automatic;
 }
@@ -4453,17 +6043,18 @@ AST_MATCHER(VarDecl, hasAutomaticStorageDuration) {
 /// int a;
 /// static int b;
 /// extern int c;
-/// varDecl(hasStaticStorageDuration())
-///   matches the function declaration y, a, b and c.
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{varDecl(hasStaticStorageDuration())}
+/// matches \match{static int y}, \match{int a}, \match{static int b} and
+/// \match{extern int c}
 AST_MATCHER(VarDecl, hasStaticStorageDuration) {
   return Node.getStorageDuration() == SD_Static;
 }
 
 /// Matches a variable declaration that has thread storage duration.
 ///
-/// Example matches z, but not x, z, or a.
-/// (matcher = varDecl(hasThreadStorageDuration())
+/// Given
 /// \code
 /// void f() {
 ///   int x;
@@ -4472,6 +6063,10 @@ AST_MATCHER(VarDecl, hasStaticStorageDuration) {
 /// }
 /// int a;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{varDecl(hasThreadStorageDuration())}
+/// matches \match{thread_local int z}
+/// but does not match \nomatch{int x} or \nomatch{type=name$a}.
 AST_MATCHER(VarDecl, hasThreadStorageDuration) {
   return Node.getStorageDuration() == SD_Thread;
 }
@@ -4479,7 +6074,7 @@ AST_MATCHER(VarDecl, hasThreadStorageDuration) {
 /// Matches a variable declaration that is an exception variable from
 /// a C++ catch block, or an Objective-C \@catch statement.
 ///
-/// Example matches x (matcher = varDecl(isExceptionVariable())
+/// Given
 /// \code
 /// void f(int y) {
 ///   try {
@@ -4487,6 +6082,9 @@ AST_MATCHER(VarDecl, hasThreadStorageDuration) {
 ///   }
 /// }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{varDecl(isExceptionVariable())}
+/// matches \match{int x}.
 AST_MATCHER(VarDecl, isExceptionVariable) {
   return Node.isExceptionVariable();
 }
@@ -4494,11 +6092,15 @@ AST_MATCHER(VarDecl, isExceptionVariable) {
 /// Checks that a call expression or a constructor call expression has
 /// a specific number of arguments (including absent default arguments).
 ///
-/// Example matches f(0, 0) (matcher = callExpr(argumentCountIs(2)))
+/// Given
 /// \code
 ///   void f(int x, int y);
-///   f(0, 0);
+///   void foo() {
+///     f(0, 0);
+///   }
 /// \endcode
+/// The matcher \matcher{callExpr(argumentCountIs(2))}
+/// matches \match{f(0, 0)}.
 AST_POLYMORPHIC_MATCHER_P(argumentCountIs,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(
                               CallExpr, CXXConstructExpr,
@@ -4518,14 +6120,17 @@ AST_POLYMORPHIC_MATCHER_P(argumentCountIs,
 /// Checks that a call expression or a constructor call expression has at least
 /// the specified number of arguments (including absent default arguments).
 ///
-/// Example matches f(0, 0) and g(0, 0, 0)
-/// (matcher = callExpr(argumentCountAtLeast(2)))
+/// Given
 /// \code
 ///   void f(int x, int y);
 ///   void g(int x, int y, int z);
-///   f(0, 0);
-///   g(0, 0, 0);
+///   void foo() {
+///     f(0, 0);
+///     g(0, 0, 0);
+///   }
 /// \endcode
+/// The matcher \matcher{callExpr(argumentCountAtLeast(2))}
+/// matches \match{f(0, 0)} and \match{g(0, 0, 0)}
 AST_POLYMORPHIC_MATCHER_P(argumentCountAtLeast,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(
                               CallExpr, CXXConstructExpr,
@@ -4545,11 +6150,13 @@ AST_POLYMORPHIC_MATCHER_P(argumentCountAtLeast,
 /// Matches the n'th argument of a call expression or a constructor
 /// call expression.
 ///
-/// Example matches y in x(y)
-///     (matcher = callExpr(hasArgument(0, declRefExpr())))
+/// Given
 /// \code
 ///   void x(int) { int y; x(y); }
 /// \endcode
+/// The matcher \matcher{callExpr(hasArgument(0, declRefExpr().bind("arg")))}
+/// matches \match{x(y)},
+/// with \matcher{type=sub$declRefExpr()} matching \match{sub="arg"$y}.
 AST_POLYMORPHIC_MATCHER_P2(hasArgument,
                            AST_POLYMORPHIC_SUPPORTED_TYPES(
                                CallExpr, CXXConstructExpr,
@@ -4565,10 +6172,7 @@ AST_POLYMORPHIC_MATCHER_P2(hasArgument,
 
 /// Matches the operand that does not contain the parameter pack.
 ///
-/// Example matches `(0 + ... + args)` and `(args * ... * 1)`
-///     (matcher = cxxFoldExpr(hasFoldInit(expr())))
-///   with hasFoldInit(...)
-///     matching `0` and `1` respectively
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4580,6 +6184,12 @@ AST_POLYMORPHIC_MATCHER_P2(hasArgument,
 ///       return (args * ... * 1);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(hasFoldInit(expr().bind("init")))}
+/// matches \match{(0 + ... + args)} and \match{(args * ... * 1)}
+/// with \matcher{type=sub$hasFoldInit(expr().bind("init"))} matching
+/// \match{sub=init$0} and \match{sub=init$1}.
 AST_MATCHER_P(CXXFoldExpr, hasFoldInit, internal::Matcher<Expr>, InnerMacher) {
   const auto *const Init = Node.getInit();
   return Init && InnerMacher.matches(*Init, Finder, Builder);
@@ -4587,10 +6197,7 @@ AST_MATCHER_P(CXXFoldExpr, hasFoldInit, internal::Matcher<Expr>, InnerMacher) {
 
 /// Matches the operand that contains the parameter pack.
 ///
-/// Example matches `(0 + ... + args)`
-///     (matcher = cxxFoldExpr(hasPattern(expr())))
-///   with hasPattern(...)
-///     matching `args`
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4602,6 +6209,12 @@ AST_MATCHER_P(CXXFoldExpr, hasFoldInit, internal::Matcher<Expr>, InnerMacher) {
 ///       return (args * ... * 1);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(hasPattern(expr().bind("pattern")))}
+/// matches \match{(0 + ... + args)} and \match{(args * ... * 1)},
+/// with \matcher{type=sub$hasPattern(expr().bind("pattern"))} matching
+/// \match{count=2;sub=pattern$args} two times.
 AST_MATCHER_P(CXXFoldExpr, hasPattern, internal::Matcher<Expr>, InnerMacher) {
   const Expr *const Pattern = Node.getPattern();
   return Pattern && InnerMacher.matches(*Pattern, Finder, Builder);
@@ -4609,8 +6222,7 @@ AST_MATCHER_P(CXXFoldExpr, hasPattern, internal::Matcher<Expr>, InnerMacher) {
 
 /// Matches right-folding fold expressions.
 ///
-/// Example matches `(args * ... * 1)`
-///     (matcher = cxxFoldExpr(isRightFold()))
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4622,12 +6234,15 @@ AST_MATCHER_P(CXXFoldExpr, hasPattern, internal::Matcher<Expr>, InnerMacher) {
 ///       return (args * ... * 1);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(isRightFold())}
+/// matches \match{(args * ... * 1)}.
 AST_MATCHER(CXXFoldExpr, isRightFold) { return Node.isRightFold(); }
 
 /// Matches left-folding fold expressions.
 ///
-/// Example matches `(0 + ... + args)`
-///     (matcher = cxxFoldExpr(isLeftFold()))
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4639,13 +6254,16 @@ AST_MATCHER(CXXFoldExpr, isRightFold) { return Node.isRightFold(); }
 ///       return (args * ... * 1);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(isLeftFold())}
+/// matches \match{(0 + ... + args)}.
 AST_MATCHER(CXXFoldExpr, isLeftFold) { return Node.isLeftFold(); }
 
 /// Matches unary fold expressions, i.e. fold expressions without an
 /// initializer.
 ///
-/// Example matches `(args * ...)`
-///     (matcher = cxxFoldExpr(isUnaryFold()))
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4657,12 +6275,15 @@ AST_MATCHER(CXXFoldExpr, isLeftFold) { return Node.isLeftFold(); }
 ///       return (args * ...);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(isUnaryFold())}
+/// matches \match{(args * ...)}, but not \nomatch{(0 + ... + args)}.
 AST_MATCHER(CXXFoldExpr, isUnaryFold) { return Node.getInit() == nullptr; }
 
 /// Matches binary fold expressions, i.e. fold expressions with an initializer.
 ///
-/// Example matches `(0 + ... + args)`
-///     (matcher = cxxFoldExpr(isBinaryFold()))
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
@@ -4674,32 +6295,42 @@ AST_MATCHER(CXXFoldExpr, isUnaryFold) { return Node.getInit() == nullptr; }
 ///       return (args * ...);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+///
+/// The matcher \matcher{cxxFoldExpr(isBinaryFold())}
+/// matches \match{(0 + ... + args)}.
 AST_MATCHER(CXXFoldExpr, isBinaryFold) { return Node.getInit() != nullptr; }
 
 /// Matches the n'th item of an initializer list expression.
 ///
-/// Example matches y.
-///     (matcher = initListExpr(hasInit(0, expr())))
+/// Given
 /// \code
-///   int x{y}.
+///   int y = 42;
+///   int x{y};
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{initListExpr(hasInit(0, expr()))}
+/// matches \match{{y}}.
 AST_MATCHER_P2(InitListExpr, hasInit, unsigned, N, internal::Matcher<Expr>,
                InnerMatcher) {
   return N < Node.getNumInits() &&
-          InnerMatcher.matches(*Node.getInit(N), Finder, Builder);
+         InnerMatcher.matches(*Node.getInit(N), Finder, Builder);
 }
 
 /// Matches declaration statements that contain a specific number of
 /// declarations.
 ///
-/// Example: Given
+/// Given
 /// \code
-///   int a, b;
-///   int c;
-///   int d = 2, e;
+///   void foo() {
+///     int a, b;
+///     int c;
+///     int d = 2, e;
+///   }
 /// \endcode
-/// declCountIs(2)
-///   matches 'int a, b;' and 'int d = 2, e;', but not 'int c;'.
+/// The matcher \matcher{declStmt(declCountIs(2))}
+/// matches \match{int a, b;} and \match{int d = 2, e;},
+/// but does not match \nomatch{int c;}
 AST_MATCHER_P(DeclStmt, declCountIs, unsigned, N) {
   return std::distance(Node.decl_begin(), Node.decl_end()) == (ptrdiff_t)N;
 }
@@ -4709,20 +6340,21 @@ AST_MATCHER_P(DeclStmt, declCountIs, unsigned, N) {
 /// Note that this does not work for global declarations because the AST
 /// breaks up multiple-declaration DeclStmt's into multiple single-declaration
 /// DeclStmt's.
-/// Example: Given non-global declarations
-/// \code
-///   int a, b = 0;
-///   int c;
-///   int d = 2, e;
-/// \endcode
-/// declStmt(containsDeclaration(
-///       0, varDecl(hasInitializer(anything()))))
-///   matches only 'int d = 2, e;', and
-/// declStmt(containsDeclaration(1, varDecl()))
+///
+/// Given non-global declarations
 /// \code
-///   matches 'int a, b = 0' as well as 'int d = 2, e;'
-///   but 'int c;' is not matched.
+///   void foo() {
+///     int a, b = 0;
+///     int c;
+///     int d = 2, e;
+///   }
 /// \endcode
+/// The matcher \matcher{declStmt(containsDeclaration(
+///       0, varDecl(hasInitializer(anything()))))}
+/// matches \match{int d = 2, e;}.
+/// The matcher \matcher{declStmt(containsDeclaration(1, varDecl()))}
+/// matches \match{int a, b = 0;} and \match{int d = 2, e;}
+/// but does not match \nomatch{int c;}.
 AST_MATCHER_P2(DeclStmt, containsDeclaration, unsigned, N,
                internal::Matcher<Decl>, InnerMatcher) {
   const unsigned NumDecls = std::distance(Node.decl_begin(), Node.decl_end());
@@ -4737,15 +6369,16 @@ AST_MATCHER_P2(DeclStmt, containsDeclaration, unsigned, N,
 ///
 /// Given
 /// \code
-///   try {
-///     // ...
-///   } catch (int) {
-///     // ...
-///   } catch (...) {
-///     // ...
+///   void foo() {
+///     try {}
+///     catch (int) {}
+///     catch (...) {}
 ///   }
 /// \endcode
-/// cxxCatchStmt(isCatchAll()) matches catch(...) but not catch(int).
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxCatchStmt(isCatchAll())}
+/// matches \match{catch (...) {}}
+/// but does not match \nomatch{catch(int)}
 AST_MATCHER(CXXCatchStmt, isCatchAll) {
   return Node.getExceptionDecl() == nullptr;
 }
@@ -4759,10 +6392,13 @@ AST_MATCHER(CXXCatchStmt, isCatchAll) {
 ///     int foo_;
 ///   };
 /// \endcode
-/// cxxRecordDecl(has(cxxConstructorDecl(
-///   hasAnyConstructorInitializer(anything())
-/// )))
-///   record matches Foo, hasAnyConstructorInitializer matches foo_(1)
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(
+///   hasAnyConstructorInitializer(cxxCtorInitializer().bind("ctor_init"))
+/// )}
+/// matches \match{Foo() : foo_(1) { }},
+/// with \matcher{type=sub$cxxCtorInitializer()}
+/// matching \match{sub=ctor_init$foo_(1)}.
 AST_MATCHER_P(CXXConstructorDecl, hasAnyConstructorInitializer,
               internal::Matcher<CXXCtorInitializer>, InnerMatcher) {
   auto MatchIt = matchesFirstInPointerRange(InnerMatcher, Node.init_begin(),
@@ -4781,10 +6417,13 @@ AST_MATCHER_P(CXXConstructorDecl, hasAnyConstructorInitializer,
 ///     int foo_;
 ///   };
 /// \endcode
-/// cxxRecordDecl(has(cxxConstructorDecl(hasAnyConstructorInitializer(
-///     forField(hasName("foo_"))))))
-///   matches Foo
-/// with forField matching foo_
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasAnyConstructorInitializer(
+///     forField(fieldDecl(hasName("foo_")).bind("field"))))}
+/// matches \match{Foo() : foo_(1) { }},
+/// with \matcher{type=sub$fieldDecl(hasName("foo_"))}
+/// matching \match{sub=field$foo_(1)}.
 AST_MATCHER_P(CXXCtorInitializer, forField,
               internal::Matcher<FieldDecl>, InnerMatcher) {
   const FieldDecl *NodeAsDecl = Node.getAnyMember();
@@ -4801,10 +6440,13 @@ AST_MATCHER_P(CXXCtorInitializer, forField,
 ///     int foo_;
 ///   };
 /// \endcode
-/// cxxRecordDecl(has(cxxConstructorDecl(hasAnyConstructorInitializer(
-///     withInitializer(integerLiteral(equals(1)))))))
-///   matches Foo
-/// with withInitializer matching (1)
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasAnyConstructorInitializer(
+///     withInitializer(integerLiteral(equals(1)).bind("literal"))))}
+/// matches \match{Foo() : foo_(1) { }},
+/// with \matcher{type=sub$integerLiteral(equals(1))} matching
+/// \match{sub=literal$1}.
 AST_MATCHER_P(CXXCtorInitializer, withInitializer,
               internal::Matcher<Expr>, InnerMatcher) {
   const Expr* NodeAsExpr = Node.getInit();
@@ -4817,14 +6459,17 @@ AST_MATCHER_P(CXXCtorInitializer, withInitializer,
 ///
 /// Given
 /// \code
+///   struct Bar { explicit Bar(const char*); };
 ///   struct Foo {
 ///     Foo() { }
 ///     Foo(int) : foo_("A") { }
-///     string foo_;
+///     Bar foo_{""};
 ///   };
 /// \endcode
-/// cxxConstructorDecl(hasAnyConstructorInitializer(isWritten()))
-///   will match Foo(int), but not Foo()
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasAnyConstructorInitializer(isWritten()))} will
+/// match \match{Foo(int) : foo_("A") { }}, but not \nomatch{Foo() { }}
 AST_MATCHER(CXXCtorInitializer, isWritten) {
   return Node.isWritten();
 }
@@ -4843,8 +6488,12 @@ AST_MATCHER(CXXCtorInitializer, isWritten) {
 ///     E() : B() {}
 ///   };
 /// \endcode
-/// cxxConstructorDecl(hasAnyConstructorInitializer(isBaseInitializer()))
-///   will match E(), but not match D(int).
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasAnyConstructorInitializer(isBaseInitializer()))}
+/// matches \match{E() : B() {}} and \match{D(int i) : I(i) {}}.
+/// The constructor of \c D is matched, because it implicitly has a constructor
+/// initializer for \c B .
 AST_MATCHER(CXXCtorInitializer, isBaseInitializer) {
   return Node.isBaseInitializer();
 }
@@ -4863,8 +6512,11 @@ AST_MATCHER(CXXCtorInitializer, isBaseInitializer) {
 ///     E() : B() {}
 ///   };
 /// \endcode
-/// cxxConstructorDecl(hasAnyConstructorInitializer(isMemberInitializer()))
-///   will match D(int), but not match E().
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasAnyConstructorInitializer(isMemberInitializer()))}
+///   will match \match{D(int i) : I(i) {}}, but not match \nomatch{E() : B()
+///   {}}.
 AST_MATCHER(CXXCtorInitializer, isMemberInitializer) {
   return Node.isMemberInitializer();
 }
@@ -4874,11 +6526,11 @@ AST_MATCHER(CXXCtorInitializer, isMemberInitializer) {
 ///
 /// Given
 /// \code
-///   void x(int, int, int) { int y; x(1, y, 42); }
+///   void x(int, int, int) { int y = 42; x(1, y, 42); }
 /// \endcode
-/// callExpr(hasAnyArgument(declRefExpr()))
-///   matches x(1, y, 42)
-/// with hasAnyArgument(...)
+/// The matcher
+/// \matcher{callExpr(hasAnyArgument(ignoringImplicit(declRefExpr())))} matches
+/// \match{x(1, y, 42)} with hasAnyArgument(...)
 ///   matching y
 ///
 /// For ObjectiveC, given
@@ -4886,8 +6538,10 @@ AST_MATCHER(CXXCtorInitializer, isMemberInitializer) {
 ///   @interface I - (void) f:(int) y; @end
 ///   void foo(I *i) { [i f:12]; }
 /// \endcode
-/// objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))
-///   matches [i f:12]
+/// \compile_args{-ObjC}
+/// The matcher
+/// \matcher{objcMessageExpr(hasAnyArgument(integerLiteral(equals(12))))}
+/// matches \match{[i f:12]}
 AST_POLYMORPHIC_MATCHER_P(hasAnyArgument,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(
                               CallExpr, CXXConstructExpr,
@@ -4916,8 +6570,12 @@ AST_POLYMORPHIC_MATCHER_P(hasAnyArgument,
 ///     auto g = [x = 1](){};
 ///   }
 /// \endcode
-/// In the matcher `lambdaExpr(hasAnyCapture(lambdaCapture()))`,
-/// `lambdaCapture()` matches `x` and `x=1`.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{lambdaExpr(hasAnyCapture(lambdaCapture().bind("capture")))},
+/// matches \match{[x](){}} and \match{[x = 1](){}},
+/// with \matcher{type=sub$lambdaCapture()} matching
+/// \match{sub=capture$x} and \match{sub=capture$x = 1}.
 extern const internal::VariadicAllOfMatcher<LambdaCapture> lambdaCapture;
 
 /// Matches any capture in a lambda expression.
@@ -4929,9 +6587,10 @@ extern const internal::VariadicAllOfMatcher<LambdaCapture> lambdaCapture;
 ///     auto f = [=](){ return t; };
 ///   }
 /// \endcode
-/// lambdaExpr(hasAnyCapture(lambdaCapture())) and
-/// lambdaExpr(hasAnyCapture(lambdaCapture(refersToVarDecl(hasName("t")))))
-///   both match `[=](){ return t; }`.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{lambdaExpr(hasAnyCapture(lambdaCapture()))} and
+/// \matcher{lambdaExpr(hasAnyCapture(lambdaCapture(capturesVar(hasName("t")))))}
+///   both match \match{[=](){ return t; }}.
 AST_MATCHER_P(LambdaExpr, hasAnyCapture, internal::Matcher<LambdaCapture>,
               InnerMatcher) {
   for (const LambdaCapture &Capture : Node.captures()) {
@@ -4956,9 +6615,13 @@ AST_MATCHER_P(LambdaExpr, hasAnyCapture, internal::Matcher<LambdaCapture>,
 ///     auto g = [x = 1](){};
 ///   }
 /// \endcode
-/// In the matcher
-/// lambdaExpr(hasAnyCapture(lambdaCapture(capturesVar(hasName("x")))),
-/// capturesVar(hasName("x")) matches `x` and `x = 1`.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{lambdaExpr(hasAnyCapture(
+///     lambdaCapture(capturesVar(hasName("x"))).bind("capture")))}
+/// matches \match{[x](){}} and \match{[x = 1](){}}, with
+/// \matcher{type=sub$lambdaCapture(capturesVar(hasName("x"))).bind("capture")}
+/// matching \match{sub=capture$x} and \match{sub=capture$x = 1}.
 AST_MATCHER_P(LambdaCapture, capturesVar, internal::Matcher<ValueDecl>,
               InnerMatcher) {
   if (!Node.capturesVariable())
@@ -4979,11 +6642,36 @@ AST_MATCHER_P(LambdaCapture, capturesVar, internal::Matcher<ValueDecl>,
 ///   }
 /// };
 /// \endcode
-/// lambdaExpr(hasAnyCapture(lambdaCapture(capturesThis())))
-///   matches `[this]() { return cc; }`.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{lambdaExpr(hasAnyCapture(lambdaCapture(capturesThis())))}
+/// matches \match{[this]() { return cc; }}.
 AST_MATCHER(LambdaCapture, capturesThis) { return Node.capturesThis(); }
 
 /// Matches a constructor call expression which uses list initialization.
+///
+/// Given
+/// \code
+///   namespace std {
+///     template <typename T>
+///     class initializer_list {
+///       const T* begin;
+///       const T* end;
+///     };
+///   }
+///   template <typename T> class vector {
+///     public: vector(std::initializer_list<T>) {}
+///   };
+///
+///   vector<int> a({ 1, 2, 3 });
+///   vector<int> b = { 4, 5 };
+///   int c[] = { 6, 7 };
+///   struct pair { int x; int y; };
+///   pair d = { 8, 9 };
+/// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++11-or-later}
+/// The matcher \matcher{cxxConstructExpr(isListInitialization())}
+/// matches \match{{ 4, 5 }}.
 AST_MATCHER(CXXConstructExpr, isListInitialization) {
   return Node.isListInitialization();
 }
@@ -4994,12 +6682,16 @@ AST_MATCHER(CXXConstructExpr, isListInitialization) {
 /// Given
 /// \code
 /// void foo() {
-///   struct point { double x; double y; };
-///   point pt[2] = { { 1.0, 2.0 } };
+///   struct Foo {
+///     double x;
+///   };
+///   auto Val = Foo();
 /// }
 /// \endcode
-/// initListExpr(has(cxxConstructExpr(requiresZeroInitialization()))
-/// will match the implicit array filler for pt[1].
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxConstructExpr(requiresZeroInitialization())}
+/// matches \match{Foo()} because the \c x member has to be zero initialized.
 AST_MATCHER(CXXConstructExpr, requiresZeroInitialization) {
   return Node.requiresZeroInitialization();
 }
@@ -5011,17 +6703,20 @@ AST_MATCHER(CXXConstructExpr, requiresZeroInitialization) {
 /// \code
 ///   class X { void f(int x) {} };
 /// \endcode
-/// cxxMethodDecl(hasParameter(0, hasType(varDecl())))
-///   matches f(int x) {}
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxMethodDecl(hasParameter(0, hasType(asString("int"))))}
+/// matches \match{void f(int x) {}}
 /// with hasParameter(...)
-///   matching int x
+/// matching int x.
 ///
 /// For ObjectiveC, given
 /// \code
 ///   @interface I - (void) f:(int) y; @end
 /// \endcode
-//
-/// the matcher objcMethodDecl(hasParameter(0, hasName("y")))
+/// \compile_args{-ObjC}
+///
+/// The matcher \matcher{objcMethodDecl(hasParameter(0, hasName("y")))}
 /// matches the declaration of method f with hasParameter
 /// matching y.
 AST_POLYMORPHIC_MATCHER_P2(hasParameter,
@@ -5046,9 +6741,13 @@ AST_POLYMORPHIC_MATCHER_P2(hasParameter,
 ///  int operator+(int);
 /// };
 /// \endcode
+/// \compile_args{-std=c++23-or-later}
 ///
-/// cxxMethodDecl(isExplicitObjectMemberFunction()) matches the first two
-/// methods but not the last two.
+/// The matcher \matcher{cxxMethodDecl(isExplicitObjectMemberFunction())}
+/// matches \match{int operator-(this A, int)} and
+/// \match{void fun(this A &&self)},
+/// but not \nomatch{static int operator()(int)} or
+/// \nomatch{int operator+(int)}.
 AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) {
   return Node.isExplicitObjectMemberFunction();
 }
@@ -5059,14 +6758,16 @@ AST_MATCHER(CXXMethodDecl, isExplicitObjectMemberFunction) {
 /// \code
 ///   void f(int i);
 ///   int y;
-///   f(y);
+///   void foo() {
+///     f(y);
+///   }
 /// \endcode
-/// callExpr(
+/// The matcher \matcher{callExpr(
 ///   forEachArgumentWithParam(
 ///     declRefExpr(to(varDecl(hasName("y")))),
 ///     parmVarDecl(hasType(isInteger()))
-/// ))
-///   matches f(y);
+/// ))}
+///   matches \match{f(y)};
 /// with declRefExpr(...)
 ///   matching int y
 /// and parmVarDecl(...)
@@ -5119,17 +6820,18 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParam,
 /// Given
 /// \code
 ///   void f(int i);
-///   int y;
-///   f(y);
-///   void (*f_ptr)(int) = f;
-///   f_ptr(y);
+///   void foo(int y) {
+///     f(y);
+///     void (*f_ptr)(int) = f;
+///     f_ptr(y);
+///   }
 /// \endcode
-/// callExpr(
+/// The matcher \matcher{callExpr(
 ///   forEachArgumentWithParamType(
 ///     declRefExpr(to(varDecl(hasName("y")))),
-///     qualType(isInteger()).bind("type)
-/// ))
-///   matches f(y) and f_ptr(y)
+///     qualType(isInteger()).bind("type")
+/// ))}
+///   matches \match{f(y)} and \match{f_ptr(y)}
 /// with declRefExpr(...)
 ///   matching int y
 /// and qualType(...)
@@ -5222,9 +6924,11 @@ AST_POLYMORPHIC_MATCHER_P2(forEachArgumentWithParamType,
 /// }
 /// \endcode
 ///
-/// ``parmVarDecl(isAtPosition(0))`` matches ``int a``.
+/// The matcher \matcher{parmVarDecl(isAtPosition(0))} matches
+/// \match{int a}.
 ///
-/// ``parmVarDecl(isAtPosition(1))`` matches ``int b``.
+/// The matcher \matcher{parmVarDecl(isAtPosition(1))}
+/// matches \match{int b}.
 AST_MATCHER_P(ParmVarDecl, isAtPosition, unsigned, N) {
   const clang::DeclContext *Context = Node.getParentFunctionOrMethod();
 
@@ -5247,27 +6951,32 @@ AST_MATCHER_P(ParmVarDecl, isAtPosition, unsigned, N) {
 /// \code
 ///   class X { void f(int x, int y, int z) {} };
 /// \endcode
-/// cxxMethodDecl(hasAnyParameter(hasName("y")))
-///   matches f(int x, int y, int z) {}
-/// with hasAnyParameter(...)
-///   matching int y
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxMethodDecl(hasAnyParameter(
+///   parmVarDecl(hasName("y")).bind("parm")))}
+/// matches \match{void f(int x, int y, int z) {}},
+/// with \matcher{type=sub$parmVarDecl(hasName("y"))}
+/// matching \match{sub=parm$int y}.
 ///
 /// For ObjectiveC, given
 /// \code
 ///   @interface I - (void) f:(int) y; @end
 /// \endcode
-//
-/// the matcher objcMethodDecl(hasAnyParameter(hasName("y")))
-/// matches the declaration of method f with hasParameter
+/// \compile_args{-ObjC}
+///
+/// the matcher \matcher{objcMethodDecl(hasAnyParameter(hasName("y")))}
+///   matches the declaration of method f with hasParameter
 /// matching y.
 ///
 /// For blocks, given
 /// \code
 ///   b = ^(int y) { printf("%d", y) };
 /// \endcode
+/// \compile_args{-ObjC}
 ///
-/// the matcher blockDecl(hasAnyParameter(hasName("y")))
-/// matches the declaration of the block b with hasParameter
+/// the matcher \matcher{blockDecl(hasAnyParameter(hasName("y")))}
+///   matches the declaration of the block b with hasParameter
 /// matching y.
 AST_POLYMORPHIC_MATCHER_P(hasAnyParameter,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
@@ -5291,12 +7000,12 @@ AST_POLYMORPHIC_MATCHER_P(hasAnyParameter,
 ///   void j(int i);
 ///   void k(int x, int y, int z, ...);
 /// \endcode
-/// functionDecl(parameterCountIs(2))
-///   matches \c g and \c h
-/// functionProtoType(parameterCountIs(2))
-///   matches \c g and \c h
-/// functionProtoType(parameterCountIs(3))
-///   matches \c k
+/// The matcher \matcher{functionDecl(parameterCountIs(2))}
+/// matches \match{void g(int i, int j) {}} and \match{void h(int i, int j)}
+/// The matcher \matcher{functionProtoType(parameterCountIs(1))}
+/// matches the type \match{type=typestr;count=2$void (int)} of \c f and \c j.
+/// The matcher \matcher{functionProtoType(parameterCountIs(3))} matches the
+/// type \match{type=typestr$void (int, int, int, ...)} of \c k.
 AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                           FunctionProtoType),
@@ -5320,14 +7029,24 @@ AST_POLYMORPHIC_MATCHER_P(parameterCountIs,
 ///   template <typename T, typename U>
 ///   void f(T&& t, U&& u) {}
 ///
-///   bool B = false;
-///   f(R, B);
+///   void foo() {
+///     bool B = false;
+///     f(R, B);
+///   }
 /// \endcode
-/// templateSpecializationType(forEachTemplateArgument(isExpr(expr())))
-///   matches twice, with expr() matching 'R * 2' and 'R * 4'
-/// functionDecl(forEachTemplateArgument(refersToType(builtinType())))
-///   matches the specialization f<unsigned, bool> twice, for 'unsigned'
-///   and 'bool'
+/// \compile_args{-fno-delayed-template-parsing;-std=c++11-or-later}
+/// The matcher
+/// \matcher{templateSpecializationType(forEachTemplateArgument(isExpr(expr().bind("t_arg"))))}
+/// matches \match{type=typestr;count=2$Matrix<int, R * 2, R * 4>} twice, with
+/// \matcher{type=sub$expr()} matching \match{sub=t_arg$R * 2} and
+/// \match{sub=t_arg$R * 4}.
+/// The matcher
+/// \matcher{functionDecl(forEachTemplateArgument(refersToType(qualType().bind("type"))))}
+/// matches the specialization of \match{count=2$void f(T&& t, U&& u) {}} twice
+/// for each of the template arguments, with \matcher{type=sub$qualType()}
+/// matching
+/// \match{sub=type;type=typestr$unsigned} and
+/// \match{sub=type;type=typestr$bool}.
 AST_POLYMORPHIC_MATCHER_P(
     forEachTemplateArgument,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -5356,23 +7075,23 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   void nope();
 ///   [[noreturn]] void a();
 ///   __attribute__((noreturn)) void b();
-///   struct c { [[noreturn]] c(); };
-/// \endcode
-/// functionDecl(isNoReturn())
-///   matches all of those except
-/// \code
-///   void nope();
+///   struct C { [[noreturn]] void c(); };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(isNoReturn())}
+/// matches \match{void a()}, \match{__attribute__((noreturn)) void b()} and
+/// \match{void c()} but does not match \nomatch{void nope()}.
 AST_MATCHER(FunctionDecl, isNoReturn) { return Node.isNoReturn(); }
 
 /// Matches the return type of a function declaration.
 ///
-/// Given:
+/// Given
 /// \code
 ///   class X { int f() { return 1; } };
 /// \endcode
-/// cxxMethodDecl(returns(asString("int")))
-///   matches int f() { return 1; }
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(returns(asString("int")))}
+/// matches \match{int f() { return 1; }}.
 AST_MATCHER_P(FunctionDecl, returns,
               internal::Matcher<QualType>, InnerMatcher) {
   return InnerMatcher.matches(Node.getReturnType(), Finder, Builder);
@@ -5380,19 +7099,21 @@ AST_MATCHER_P(FunctionDecl, returns,
 
 /// Matches extern "C" function or variable declarations.
 ///
-/// Given:
+/// Given
 /// \code
 ///   extern "C" void f() {}
 ///   extern "C" { void g() {} }
 ///   void h() {}
 ///   extern "C" int x = 1;
-///   extern "C" int y = 2;
+///   extern "C" { int y = 2; }
 ///   int z = 3;
 /// \endcode
-/// functionDecl(isExternC())
-///   matches the declaration of f and g, but not the declaration of h.
-/// varDecl(isExternC())
-///   matches the declaration of x and y, but not the declaration of z.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(isExternC())}
+/// matches \match{void f() {}} and \match{void g() {}}.
+/// The matcher \matcher{varDecl(isExternC())}
+/// matches \match{int x = 1} and \match{int y = 2}, but does not
+/// match \nomatch{int z = 3}.
 AST_POLYMORPHIC_MATCHER(isExternC, AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                                    VarDecl)) {
   return Node.isExternC();
@@ -5401,17 +7122,17 @@ AST_POLYMORPHIC_MATCHER(isExternC, AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
 /// Matches variable/function declarations that have "static" storage
 /// class specifier ("static" keyword) written in the source.
 ///
-/// Given:
+/// Given
 /// \code
 ///   static void f() {}
 ///   static int i = 0;
 ///   extern int j;
 ///   int k;
 /// \endcode
-/// functionDecl(isStaticStorageClass())
-///   matches the function declaration f.
-/// varDecl(isStaticStorageClass())
-///   matches the variable declaration i.
+/// The matcher \matcher{functionDecl(isStaticStorageClass())}
+/// matches \match{static void f() {}}.
+/// The matcher \matcher{varDecl(isStaticStorageClass())}
+/// matches \match{static int i = 0}.
 AST_POLYMORPHIC_MATCHER(isStaticStorageClass,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                         VarDecl)) {
@@ -5420,56 +7141,77 @@ AST_POLYMORPHIC_MATCHER(isStaticStorageClass,
 
 /// Matches deleted function declarations.
 ///
-/// Given:
+/// Given
 /// \code
 ///   void Func();
 ///   void DeletedFunc() = delete;
 /// \endcode
-/// functionDecl(isDeleted())
-///   matches the declaration of DeletedFunc, but not Func.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(isDeleted())}
+/// matches \match{void DeletedFunc()},
+/// but does not match \nomatch{void Func()}.
 AST_MATCHER(FunctionDecl, isDeleted) {
   return Node.isDeleted();
 }
 
 /// Matches defaulted function declarations.
 ///
-/// Given:
+/// Given
 /// \code
 ///   class A { ~A(); };
 ///   class B { ~B() = default; };
 /// \endcode
-/// functionDecl(isDefaulted())
-///   matches the declaration of ~B, but not ~A.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(isDefaulted())}
+///   matches \match{~B() = default},
+/// but does not match \nomatch{~A()}.
 AST_MATCHER(FunctionDecl, isDefaulted) {
   return Node.isDefaulted();
 }
 
 /// Matches weak function declarations.
 ///
-/// Given:
+/// Given
 /// \code
-///   void foo() __attribute__((__weakref__("__foo")));
-///   void bar();
+///   static void f();
+///   void g() __attribute__((weak));
 /// \endcode
-/// functionDecl(isWeak())
-///   matches the weak declaration "foo", but not "bar".
+/// The matcher \matcher{functionDecl(isWeak())}
+///   matches the weak declaration
+/// \match{void g() __attribute__((weak))},
+/// but does not match \nomatch{static void foo_v1()}.
 AST_MATCHER(FunctionDecl, isWeak) { return Node.isWeak(); }
 
 /// Matches functions that have a dynamic exception specification.
 ///
-/// Given:
+/// Given
 /// \code
-///   void f();
-///   void g() noexcept;
-///   void h() noexcept(true);
-///   void i() noexcept(false);
-///   void j() throw();
-///   void k() throw(int);
-///   void l() throw(...);
-/// \endcode
-/// functionDecl(hasDynamicExceptionSpec()) and
-///   functionProtoType(hasDynamicExceptionSpec())
-///   match the declarations of j, k, and l, but not f, g, h, or i.
+///   void f(int);
+///   void g(int) noexcept;
+///   void h(int) noexcept(true);
+///   void i(int) noexcept(false);
+///   void j(int) throw();
+///   void k(int) throw(int);
+///   void l(int) throw(...);
+/// \endcode
+/// \compile_args{-std=c++11-14}
+/// The matcher \matcher{functionDecl(hasDynamicExceptionSpec())}
+/// matches the declarations \match{void j(int) throw()},
+/// \match{void k(int) throw(int)}
+/// and \match{void l(int) throw(...)},
+/// but does not match \nomatch{void f(int)}, \nomatch{void g(int) noexcept},
+/// \nomatch{void h(int) noexcept(true)}
+/// or \nomatch{void i(int) noexcept(true)}.
+/// The matcher
+/// \matcher{functionProtoType(hasDynamicExceptionSpec())} matches
+/// the type \match{type=typestr$void (int) throw()} of \c j ,
+/// the type \match{type=typestr$void (int) throw(int)} of \c k and
+/// the type \match{type=typestr$void (int) throw(...)} of \c l .
+/// It does not match
+/// the type \nomatch{type=typestr$void (int) noexcept} of \c f ,
+/// the type \nomatch{type=typestr$void (int) noexcept} of \c g ,
+/// the type \nomatch{type=typestr$void (int) noexcept(int)} of \c h or
+/// the type \nomatch{type=typestr$void (int) noexcept(...)} of \c i .
 AST_POLYMORPHIC_MATCHER(hasDynamicExceptionSpec,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                         FunctionProtoType)) {
@@ -5480,16 +7222,29 @@ AST_POLYMORPHIC_MATCHER(hasDynamicExceptionSpec,
 
 /// Matches functions that have a non-throwing exception specification.
 ///
-/// Given:
+/// Given
 /// \code
-///   void f();
-///   void g() noexcept;
-///   void h() throw();
-///   void i() throw(int);
-///   void j() noexcept(false);
+///   void f(int);
+///   void g(int) noexcept;
+///   void h(int) noexcept(false);
+///   void i(int) throw();
+///   void j(int) throw(int);
 /// \endcode
-/// functionDecl(isNoThrow()) and functionProtoType(isNoThrow())
-///   match the declarations of g, and h, but not f, i or j.
+/// \compile_args{-std=c++11-14}
+/// The matcher \matcher{functionDecl(isNoThrow())}
+/// matches the declaration \match{void g(int) noexcept}
+/// and \match{void i(int) throw()},
+/// but does not match \nomatch{void f(int)},
+/// \nomatch{void h(int) noexcept(false)}
+/// or \nomatch{void j(int) throw(int)}.
+/// The matcher
+/// \matcher{functionProtoType(isNoThrow())}
+/// matches the type \match{type=typestr$void (int) throw()} of \c i
+/// and the type \match{type=typestr$void (int) noexcept} of \c g,
+/// but does not match
+/// the type \nomatch{type=typestr$void (int)} of \c f ,
+/// the type \nomatch{type=typestr$void (int) noexcept(false)} of \c h or
+/// the type \nomatch{type=typestr$void (int) throw(int)} of \c j .
 AST_POLYMORPHIC_MATCHER(isNoThrow,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl,
                                                         FunctionProtoType)) {
@@ -5511,17 +7266,20 @@ AST_POLYMORPHIC_MATCHER(isNoThrow,
 /// Matches consteval function declarations and if consteval/if ! consteval
 /// statements.
 ///
-/// Given:
+/// Given
 /// \code
 ///   consteval int a();
 ///   void b() { if consteval {} }
 ///   void c() { if ! consteval {} }
 ///   void d() { if ! consteval {} else {} }
 /// \endcode
-/// functionDecl(isConsteval())
-///   matches the declaration of "int a()".
-/// ifStmt(isConsteval())
-///   matches the if statement in "void b()", "void c()", "void d()".
+/// \compile_args{-std=c++23-or-later}
+/// The matcher \matcher{functionDecl(isConsteval())}
+/// matches \match{consteval int a()}.
+/// The matcher \matcher{ifStmt(isConsteval())}
+/// matches the if statements
+/// \match{if consteval {}}, \match{if ! consteval {}} and
+/// \match{if ! consteval {} else {}}.
 AST_POLYMORPHIC_MATCHER(isConsteval,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(FunctionDecl, IfStmt)) {
   return Node.isConsteval();
@@ -5530,18 +7288,19 @@ AST_POLYMORPHIC_MATCHER(isConsteval,
 /// Matches constexpr variable and function declarations,
 ///        and if constexpr.
 ///
-/// Given:
+/// Given
 /// \code
 ///   constexpr int foo = 42;
 ///   constexpr int bar();
 ///   void baz() { if constexpr(1 > 0) {} }
 /// \endcode
-/// varDecl(isConstexpr())
-///   matches the declaration of foo.
-/// functionDecl(isConstexpr())
-///   matches the declaration of bar.
-/// ifStmt(isConstexpr())
-///   matches the if statement in baz.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{varDecl(isConstexpr())}
+/// matches \match{constexpr int foo = 42}.
+/// The matcher \matcher{functionDecl(isConstexpr())}
+/// matches \match{constexpr int bar()}.
+/// The matcher \matcher{ifStmt(isConstexpr())}
+/// matches \match{if constexpr(1 > 0) {}}.
 AST_POLYMORPHIC_MATCHER(isConstexpr,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(VarDecl,
                                                         FunctionDecl,
@@ -5551,15 +7310,19 @@ AST_POLYMORPHIC_MATCHER(isConstexpr,
 
 /// Matches constinit variable declarations.
 ///
-/// Given:
+/// Given
 /// \code
 ///   constinit int foo = 42;
 ///   constinit const char* bar = "bar";
 ///   int baz = 42;
 ///   [[clang::require_constant_initialization]] int xyz = 42;
 /// \endcode
-/// varDecl(isConstinit())
-///   matches the declaration of `foo` and `bar`, but not `baz` and `xyz`.
+/// \compile_args{-std=c++20-or-later}
+/// The matcher \matcher{varDecl(isConstinit())}
+/// matches the declaration of \match{constinit int foo = 42}
+/// and \match{constinit const char* bar = "bar"},
+/// but does not match \nomatch{int baz = 42} or
+/// \nomatch{[[clang::require_constant_initialization]] int xyz = 42}.
 AST_MATCHER(VarDecl, isConstinit) {
   if (const auto *CIA = Node.getAttr<ConstInitAttr>())
     return CIA->isConstinit();
@@ -5568,8 +7331,11 @@ AST_MATCHER(VarDecl, isConstinit) {
 
 /// Matches selection statements with initializer.
 ///
-/// Given:
+/// Given
 /// \code
+///  struct vec { int* begin(); int* end(); };
+///  int foobar();
+///  vec& get_range();
 ///  void foo() {
 ///    if (int i = foobar(); i > 0) {}
 ///    switch (int i = foobar(); i) {}
@@ -5581,12 +7347,17 @@ AST_MATCHER(VarDecl, isConstinit) {
 ///    for (auto& x : get_range()) {}
 ///  }
 /// \endcode
-/// ifStmt(hasInitStatement(anything()))
-///   matches the if statement in foo but not in bar.
-/// switchStmt(hasInitStatement(anything()))
-///   matches the switch statement in foo but not in bar.
-/// cxxForRangeStmt(hasInitStatement(anything()))
-///   matches the range for statement in foo but not in bar.
+/// \compile_args{-std=c++20-or-later}
+/// The matcher \matcher{ifStmt(hasInitStatement(anything()))}
+///   matches the if statement \match{if (int i = foobar(); i > 0) {}}
+///   in foo but not \nomatch{if (foobar() > 0) {}} in bar.
+/// The matcher \matcher{switchStmt(hasInitStatement(anything()))}
+///   matches the switch statement \match{switch (int i = foobar(); i) {}}
+///   in foo but not \nomatch{switch (foobar()) {}} in bar.
+/// The matcher \matcher{cxxForRangeStmt(hasInitStatement(anything()))}
+///   matches the range for statement
+///   \match{for (auto& a = get_range(); auto& x : a) {}} in foo
+///   but not \nomatch{for (auto& x : get_range()) {}} in bar.
 AST_POLYMORPHIC_MATCHER_P(hasInitStatement,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(IfStmt, SwitchStmt,
                                                           CXXForRangeStmt),
@@ -5598,10 +7369,15 @@ AST_POLYMORPHIC_MATCHER_P(hasInitStatement,
 /// Matches the condition expression of an if statement, for loop,
 /// switch statement or conditional operator.
 ///
-/// Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
+/// Given
 /// \code
+/// void foo() {
 ///   if (true) {}
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{ifStmt(hasCondition(cxxBoolLiteral(equals(true))))}
+/// \match{if (true) {}}
 AST_POLYMORPHIC_MATCHER_P(
     hasCondition,
     AST_POLYMORPHIC_SUPPORTED_TYPES(IfStmt, ForStmt, WhileStmt, DoStmt,
@@ -5614,11 +7390,15 @@ AST_POLYMORPHIC_MATCHER_P(
 
 /// Matches the then-statement of an if statement.
 ///
-/// Examples matches the if statement
-///   (matcher = ifStmt(hasThen(cxxBoolLiteral(equals(true)))))
+/// Given
 /// \code
+/// void foo() {
 ///   if (false) true; else false;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{ifStmt(hasThen(cxxBoolLiteral(equals(true))))}
+/// \match{if (false) true; else false}
 AST_MATCHER_P(IfStmt, hasThen, internal::Matcher<Stmt>, InnerMatcher) {
   const Stmt *const Then = Node.getThen();
   return (Then != nullptr && InnerMatcher.matches(*Then, Finder, Builder));
@@ -5626,11 +7406,15 @@ AST_MATCHER_P(IfStmt, hasThen, internal::Matcher<Stmt>, InnerMatcher) {
 
 /// Matches the else-statement of an if statement.
 ///
-/// Examples matches the if statement
-///   (matcher = ifStmt(hasElse(cxxBoolLiteral(equals(true)))))
+/// Given
 /// \code
+/// void foo() {
 ///   if (false) false; else true;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{ifStmt(hasElse(cxxBoolLiteral(equals(true))))}
+/// \match{if (false) false; else true}
 AST_MATCHER_P(IfStmt, hasElse, internal::Matcher<Stmt>, InnerMatcher) {
   const Stmt *const Else = Node.getElse();
   return (Else != nullptr && InnerMatcher.matches(*Else, Finder, Builder));
@@ -5644,10 +7428,12 @@ AST_MATCHER_P(IfStmt, hasElse, internal::Matcher<Stmt>, InnerMatcher) {
 /// \code
 ///   class X { int a; int b; };
 /// \endcode
-/// cxxRecordDecl(
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(
 ///     has(fieldDecl(hasName("a"), hasType(type().bind("t")))),
-///     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))
-///   matches the class \c X, as \c a and \c b have the same type.
+///     has(fieldDecl(hasName("b"), hasType(type(equalsBoundNode("t"))))))}
+///   matches \match{class X { int a; int b; }}, as \c a and \c b have the same
+///   type.
 ///
 /// Note that when multiple matches are involved via \c forEach* matchers,
 /// \c equalsBoundNodes acts as a filter.
@@ -5677,14 +7463,19 @@ AST_POLYMORPHIC_MATCHER_P(equalsBoundNode,
 ///
 /// Given
 /// \code
+/// struct A {};
+/// A* GetAPointer();
+/// void foo() {
 ///   if (A* a = GetAPointer()) {}
+/// }
 /// \endcode
-/// hasConditionVariableStatement(...)
-///   matches 'A* a = GetAPointer()'.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{ifStmt(hasConditionVariableStatement(declStmt()))}
+/// \match{if (A* a = GetAPointer()) {}}
 AST_MATCHER_P(IfStmt, hasConditionVariableStatement,
               internal::Matcher<DeclStmt>, InnerMatcher) {
-  const DeclStmt* const DeclarationStatement =
-    Node.getConditionVariableDeclStmt();
+  const DeclStmt *const DeclarationStatement =
+      Node.getConditionVariableDeclStmt();
   return DeclarationStatement != nullptr &&
          InnerMatcher.matches(*DeclarationStatement, Finder, Builder);
 }
@@ -5696,11 +7487,11 @@ AST_MATCHER_P(IfStmt, hasConditionVariableStatement,
 ///   int i[5];
 ///   void f() { i[1] = 42; }
 /// \endcode
-/// arraySubscriptExpression(hasIndex(integerLiteral()))
-///   matches \c i[1] with the \c integerLiteral() matching \c 1
-AST_MATCHER_P(ArraySubscriptExpr, hasIndex,
-              internal::Matcher<Expr>, InnerMatcher) {
-  if (const Expr* Expression = Node.getIdx())
+/// The matcher \matcher{arraySubscriptExpr(hasIndex(integerLiteral()))}
+///   matches \match{i[1]} with the \c integerLiteral() matching \c 1
+AST_MATCHER_P(ArraySubscriptExpr, hasIndex, internal::Matcher<Expr>,
+              InnerMatcher) {
+  if (const Expr *Expression = Node.getIdx())
     return InnerMatcher.matches(*Expression, Finder, Builder);
   return false;
 }
@@ -5712,9 +7503,9 @@ AST_MATCHER_P(ArraySubscriptExpr, hasIndex,
 ///   int i[5];
 ///   void f() { i[1] = 42; }
 /// \endcode
-/// arraySubscriptExpression(hasBase(implicitCastExpr(
-///     hasSourceExpression(declRefExpr()))))
-///   matches \c i[1] with the \c declRefExpr() matching \c i
+/// The matcher \matcher{arraySubscriptExpr(hasBase(implicitCastExpr(
+///     hasSourceExpression(declRefExpr()))))}
+///   matches \match{i[1]} with the \c declRefExpr() matching \c i
 AST_MATCHER_P(ArraySubscriptExpr, hasBase,
               internal::Matcher<Expr>, InnerMatcher) {
   if (const Expr* Expression = Node.getBase())
@@ -5729,23 +7520,25 @@ AST_MATCHER_P(ArraySubscriptExpr, hasBase,
 ///
 /// Given
 /// \code
+/// void foo() {
 ///   for (;;) {}
+/// }
 /// \endcode
-/// forStmt(hasBody(compoundStmt()))
-///   matches 'for (;;) {}'
-/// with compoundStmt()
-///   matching '{}'
+/// The matcher \matcher{forStmt(hasBody(compoundStmt().bind("body")))}
+/// matches \match{for (;;) {}}
+/// with \matcher{type=sub$compoundStmt()}
+///   matching \match{sub=body${}}
 ///
 /// Given
 /// \code
 ///   void f();
 ///   void f() {}
 /// \endcode
-/// functionDecl(hasBody(compoundStmt()))
-///   matches 'void f() {}'
-/// with compoundStmt()
-///   matching '{}'
-///   but does not match 'void f();'
+/// The matcher \matcher{functionDecl(hasBody(compoundStmt().bind("compound")))}
+/// \match{void f() {}}
+/// with \matcher{type=sub$compoundStmt()}
+/// matching \match{sub=compound${}},
+/// but it does not match \nomatch{void f();}.
 AST_POLYMORPHIC_MATCHER_P(
     hasBody,
     AST_POLYMORPHIC_SUPPORTED_TYPES(DoStmt, ForStmt, WhileStmt, CXXForRangeStmt,
@@ -5768,31 +7561,28 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   void f() {}
 ///   void g();
 /// \endcode
-/// functionDecl(hasAnyBody(compoundStmt()))
-///   matches both 'void f();'
-///   and 'void f() {}'
-/// with compoundStmt()
-///   matching '{}'
-///   but does not match 'void g();'
-AST_MATCHER_P(FunctionDecl, hasAnyBody,
-              internal::Matcher<Stmt>, InnerMatcher) {
+/// The matcher \matcher{functionDecl(hasAnyBody(compoundStmt().bind("body")))}
+/// matches \match{void f() {}} and the declaration \match{void f()},
+/// with \matcher{type=sub$compoundStmt()} matching \match{sub=body${}}, but it
+/// does not match \nomatch{void g()}.
+AST_MATCHER_P(FunctionDecl, hasAnyBody, internal::Matcher<Stmt>, InnerMatcher) {
   const Stmt *const Statement = Node.getBody();
   return (Statement != nullptr &&
           InnerMatcher.matches(*Statement, Finder, Builder));
 }
 
-
 /// Matches compound statements where at least one substatement matches
 /// a given matcher. Also matches StmtExprs that have CompoundStmt as children.
 ///
 /// Given
 /// \code
-///   { {}; 1+2; }
+/// void foo() { { {}; 1+2; } }
 /// \endcode
-/// hasAnySubstatement(compoundStmt())
-///   matches '{ {}; 1+2; }'
-/// with compoundStmt()
-///   matching '{}'
+/// The matcher
+/// \matcher{compoundStmt(hasAnySubstatement(compoundStmt().bind("compound")))}
+/// \match{{ {}; 1+2; }} and \match{{ { {}; 1+2; } }}
+/// with \matcher{type=sub$compoundStmt()}
+/// matching \match{sub=compound${}} and \match{sub=compound${ {}; 1+2; }}.
 AST_POLYMORPHIC_MATCHER_P(hasAnySubstatement,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(CompoundStmt,
                                                           StmtExpr),
@@ -5808,10 +7598,12 @@ AST_POLYMORPHIC_MATCHER_P(hasAnySubstatement,
 ///
 /// Example: Given
 /// \code
+/// void foo() {
 ///   { for (;;) {} }
+/// }
 /// \endcode
-/// compoundStmt(statementCountIs(0)))
-///   matches '{}'
+/// The matcher \matcher{compoundStmt(statementCountIs(0))}
+/// \match{{}}
 ///   but does not match the outer compound statement.
 AST_MATCHER_P(CompoundStmt, statementCountIs, unsigned, N) {
   return Node.size() == N;
@@ -5821,24 +7613,31 @@ AST_MATCHER_P(CompoundStmt, statementCountIs, unsigned, N) {
 ///
 /// Given
 /// \code
+/// void f(char, bool, double, int);
+/// void foo() {
 ///   f('\0', false, 3.14, 42);
+/// }
 /// \endcode
-/// characterLiteral(equals(0))
-///   matches '\0'
-/// cxxBoolLiteral(equals(false)) and cxxBoolLiteral(equals(0))
-///   match false
-/// floatLiteral(equals(3.14)) and floatLiteral(equals(314e-2))
-///   match 3.14
-/// integerLiteral(equals(42))
-///   matches 42
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{characterLiteral(equals(0U))} matches \match{'\0'}.
+/// The matchers \matcher{cxxBoolLiteral(equals(false))} and
+/// \matcher{cxxBoolLiteral(equals(0))} match \match{false}.
+/// The matcher \matcher{floatLiteral(equals(3.14))} matches \match{3.14}.
+/// The matcher \matcher{integerLiteral(equals(42))} matches \match{42}.
 ///
 /// Note that you cannot directly match a negative numeric literal because the
 /// minus sign is not part of the literal: It is a unary operator whose operand
 /// is the positive numeric literal. Instead, you must use a unaryOperator()
 /// matcher to match the minus sign:
 ///
-/// unaryOperator(hasOperatorName("-"),
-///               hasUnaryOperand(integerLiteral(equals(13))))
+/// Given
+/// \code
+///   int val = -1;
+/// \endcode
+///
+/// The matcher \matcher{unaryOperator(hasOperatorName("-"),
+///               hasUnaryOperand(integerLiteral(equals(1))))}
+/// matches \match{-1}.
 ///
 /// Usable as: Matcher<CharacterLiteral>, Matcher<CXXBoolLiteralExpr>,
 ///            Matcher<FloatingLiteral>, Matcher<IntegerLiteral>
@@ -5882,19 +7681,26 @@ AST_POLYMORPHIC_MATCHER_P_OVERLOAD(equals,
 /// Matches the operator Name of operator expressions and fold expressions
 /// (binary or unary).
 ///
-/// Example matches a || b (matcher = binaryOperator(hasOperatorName("||")))
+/// Given
 /// \code
-///   !(a || b)
+//// void foo(bool a, bool b) {
+///   !(a || b);
+///  }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{binaryOperator(hasOperatorName("||"))}
+/// matches \match{a || b}
 ///
-/// Example matches `(0 + ... + args)`
-///     (matcher = cxxFoldExpr(hasOperatorName("+")))
+/// Given
 /// \code
 ///   template <typename... Args>
 ///   auto sum(Args... args) {
 ///       return (0 + ... + args);
 ///   }
 /// \endcode
+/// \compile_args{-std=c++17-or-later}
+/// The matcher \matcher{cxxFoldExpr(hasOperatorName("+"))}
+///  matches \match{(0 + ... + args)}.
 AST_POLYMORPHIC_MATCHER_P(
     hasOperatorName,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -5909,9 +7715,28 @@ AST_POLYMORPHIC_MATCHER_P(
 /// Matches operator expressions (binary or unary) that have any of the
 /// specified names.
 ///
-///    hasAnyOperatorName("+", "-")
-///  Is equivalent to
-///    anyOf(hasOperatorName("+"), hasOperatorName("-"))
+/// It provides a compact way of writing if an operator has any of the specified
+/// names:
+/// The matcher
+///    \c hasAnyOperatorName("+", "-")
+/// Is equivalent to
+///    \c{anyOf(hasOperatorName("+"), hasOperatorName("-"))}
+///
+/// Given
+/// \code
+//// void foo(bool a, bool b) {
+///   !(a || b);
+///  }
+///
+//// void bar(bool a, bool b) {
+///   a && b;
+///  }
+/// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{binaryOperator(hasAnyOperatorName("||", "&&"))}
+/// matches \match{a || b} and \match{a && b}.
+/// The matcher \matcher{unaryOperator(hasAnyOperatorName("-", "!"))}
+/// matches \match{!(a || b)}.
 extern const internal::VariadicFunction<
     internal::PolymorphicMatcher<internal::HasAnyOperatorNameMatcher,
                                  AST_POLYMORPHIC_SUPPORTED_TYPES(
@@ -5923,18 +7748,24 @@ extern const internal::VariadicFunction<
 
 /// Matches all kinds of assignment operators.
 ///
-/// Example 1: matches a += b (matcher = binaryOperator(isAssignmentOperator()))
+/// Given
 /// \code
+/// void foo(int a, int b) {
 ///   if (a == b)
 ///     a += b;
+/// }
 /// \endcode
+/// The matcher \matcher{binaryOperator(isAssignmentOperator())}
+/// matches \match{a += b}.
 ///
-/// Example 2: matches s1 = s2
-///            (matcher = cxxOperatorCallExpr(isAssignmentOperator()))
+/// Given
 /// \code
 ///   struct S { S& operator=(const S&); };
 ///   void x() { S s1, s2; s1 = s2; }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxOperatorCallExpr(isAssignmentOperator())}
+/// matches \match{s1 = s2}.
 AST_POLYMORPHIC_MATCHER(
     isAssignmentOperator,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -5944,18 +7775,24 @@ AST_POLYMORPHIC_MATCHER(
 
 /// Matches comparison operators.
 ///
-/// Example 1: matches a == b (matcher = binaryOperator(isComparisonOperator()))
+/// Given
 /// \code
+/// void foo(int a, int b) {
 ///   if (a == b)
 ///     a += b;
+/// }
 /// \endcode
+/// The matcher \matcher{binaryOperator(isComparisonOperator())}
+/// matches \match{a == b}
 ///
-/// Example 2: matches s1 < s2
-///            (matcher = cxxOperatorCallExpr(isComparisonOperator()))
+/// Given
 /// \code
 ///   struct S { bool operator<(const S& other); };
 ///   void x(S s1, S s2) { bool b1 = s1 < s2; }
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxOperatorCallExpr(isComparisonOperator())}
+/// matches \match{s1 < s2}
 AST_POLYMORPHIC_MATCHER(
     isComparisonOperator,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -5965,10 +7802,17 @@ AST_POLYMORPHIC_MATCHER(
 
 /// Matches the left hand side of binary operator expressions.
 ///
-/// Example matches a (matcher = binaryOperator(hasLHS()))
+/// Given
 /// \code
-///   a || b
+/// void foo(bool a, bool b) {
+///   a || b;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{binaryOperator(hasLHS(expr().bind("lhs")))}
+/// matches \match{a || b},
+/// with \matcher{type=sub$expr()}
+/// matching \match{sub=lhs$a}.
 AST_POLYMORPHIC_MATCHER_P(
     hasLHS,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -5982,10 +7826,17 @@ AST_POLYMORPHIC_MATCHER_P(
 
 /// Matches the right hand side of binary operator expressions.
 ///
-/// Example matches b (matcher = binaryOperator(hasRHS()))
+/// Given
 /// \code
-///   a || b
+/// void foo(bool a, bool b) {
+///   a || b;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{binaryOperator(hasRHS(expr().bind("rhs")))}
+/// matches \match{a || b},
+/// with \matcher{type=sub$expr()}
+/// matching \match{sub=rhs$b}.
 AST_POLYMORPHIC_MATCHER_P(
     hasRHS,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -5999,6 +7850,32 @@ AST_POLYMORPHIC_MATCHER_P(
 
 /// Matches if either the left hand side or the right hand side of a
 /// binary operator or fold expression matches.
+///
+/// Given
+/// \code
+///   struct S {};
+///   bool operator ==(const S&, const S&);
+///
+///   void f(int a, const S&lhs, const S&rhs) {
+///       a + 0;
+///       lhs == rhs;
+///       lhs != rhs;
+///   }
+///
+///   template <typename ...Ts>
+///   auto sum(Ts... args) {
+///     return (0 + ... + args);
+///   }
+/// \endcode
+/// \compile_args{-std=c++20-or-later}
+///
+/// The matcher \matcher{binaryOperator(hasEitherOperand(integerLiteral()))}
+/// matches \match{a + 0}.
+/// The matcher \matcher{cxxOperatorCallExpr(hasEitherOperand(declRefExpr(to(
+/// parmVarDecl(hasName("lhs"))))))} matches \match{lhs == rhs} and
+/// \match{lhs != rhs}.
+/// The matcher \matcher{cxxFoldExpr(hasEitherOperand(integerLiteral()))}
+/// matches \match{(0 + ... + args)}.
 AST_POLYMORPHIC_MATCHER_P(
     hasEitherOperand,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -6012,14 +7889,20 @@ AST_POLYMORPHIC_MATCHER_P(
 /// Matches if both matchers match with opposite sides of the binary operator
 /// or fold expression.
 ///
-/// Example matcher = binaryOperator(hasOperands(integerLiteral(equals(1),
-///                                              integerLiteral(equals(2)))
+/// Given
 /// \code
-///   1 + 2 // Match
-///   2 + 1 // Match
-///   1 + 1 // No match
-///   2 + 2 // No match
+/// void foo() {
+///   1 + 2; // Match
+///   2 + 1; // Match
+///   1 + 1; // No match
+///   2 + 2; // No match
+/// }
 /// \endcode
+/// The matcher \matcher{binaryOperator(hasOperands(integerLiteral(equals(1)),
+///                                             integerLiteral(equals(2))))}
+/// matches \match{1 + 2} and \match{2 + 1},
+/// but does not match \nomatch{1 + 1}
+/// or \nomatch{2 + 2}.
 AST_POLYMORPHIC_MATCHER_P2(
     hasOperands,
     AST_POLYMORPHIC_SUPPORTED_TYPES(BinaryOperator, CXXOperatorCallExpr,
@@ -6033,11 +7916,15 @@ AST_POLYMORPHIC_MATCHER_P2(
 
 /// Matches if the operand of a unary operator matches.
 ///
-/// Example matches true (matcher = hasUnaryOperand(
-///                                   cxxBoolLiteral(equals(true))))
 /// \code
-///   !true
+/// void foo() {
+///   !true;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher
+/// \matcher{unaryOperator(hasUnaryOperand(cxxBoolLiteral(equals(true))))}
+/// matches \match{!true}.
 AST_POLYMORPHIC_MATCHER_P(hasUnaryOperand,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(UnaryOperator,
                                                           CXXOperatorCallExpr),
@@ -6050,18 +7937,27 @@ AST_POLYMORPHIC_MATCHER_P(hasUnaryOperand,
 /// Matches if the cast's source expression
 /// or opaque value's source expression matches the given matcher.
 ///
-/// Example 1: matches "a string"
-/// (matcher = castExpr(hasSourceExpression(cxxConstructExpr())))
+/// Given
 /// \code
-/// class URL { URL(string); };
-/// URL url = "a string";
+///  struct URL { URL(const char*); };
+///  URL url = "a string";
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{castExpr(hasSourceExpression(cxxConstructExpr()))}
+/// matches \match{"a string"}.
 ///
-/// Example 2: matches 'b' (matcher =
-/// opaqueValueExpr(hasSourceExpression(implicitCastExpr(declRefExpr())))
+/// Given
 /// \code
-/// int a = b ?: 1;
+/// void foo(bool b) {
+///   int a = b ?: 1;
+/// }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher
+/// \matcher{opaqueValueExpr(hasSourceExpression(
+///               implicitCastExpr(has(
+///                 implicitCastExpr(has(declRefExpr()))))))}
+/// matches \match{count=2$b} twice, for the condition and the true expression.
 AST_POLYMORPHIC_MATCHER_P(hasSourceExpression,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(CastExpr,
                                                           OpaqueValueExpr),
@@ -6074,11 +7970,12 @@ AST_POLYMORPHIC_MATCHER_P(hasSourceExpression,
 
 /// Matches casts that has a given cast kind.
 ///
-/// Example: matches the implicit cast around \c 0
-/// (matcher = castExpr(hasCastKind(CK_NullToPointer)))
+/// Given
 /// \code
 ///   int *p = 0;
 /// \endcode
+/// The matcher \matcher{castExpr(hasCastKind(CK_NullToPointer))}
+/// matches the implicit cast around \match{0}
 ///
 /// If the matcher is use from clang-query, CastKind parameter
 /// should be passed as a quoted string. e.g., hasCastKind("CK_NullToPointer").
@@ -6090,14 +7987,30 @@ AST_MATCHER_P(CastExpr, hasCastKind, CastKind, Kind) {
 ///
 /// (Note: Clang's AST refers to other conversions as "casts" too, and calls
 /// actual casts "explicit" casts.)
-AST_MATCHER_P(ExplicitCastExpr, hasDestinationType,
-              internal::Matcher<QualType>, InnerMatcher) {
+///
+/// \code
+///   unsigned int a = (unsigned int)0;
+/// \endcode
+///
+/// The matcher \matcher{explicitCastExpr(hasDestinationType(
+/// qualType(isUnsignedInteger())))} matches \match{(unsigned int)0}.
+AST_MATCHER_P(ExplicitCastExpr, hasDestinationType, internal::Matcher<QualType>,
+              InnerMatcher) {
   const QualType NodeType = Node.getTypeAsWritten();
   return InnerMatcher.matches(NodeType, Finder, Builder);
 }
 
 /// Matches implicit casts whose destination type matches a given
 /// matcher.
+///
+/// Given
+/// \code
+///   unsigned int a = 0;
+/// \endcode
+///
+/// The matcher
+/// \matcher{implicitCastExpr(hasImplicitDestinationType(
+/// qualType(isUnsignedInteger())))} matches \match{0}.
 AST_MATCHER_P(ImplicitCastExpr, hasImplicitDestinationType,
               internal::Matcher<QualType>, InnerMatcher) {
   return InnerMatcher.matches(Node.getType(), Finder, Builder);
@@ -6107,50 +8020,66 @@ AST_MATCHER_P(ImplicitCastExpr, hasImplicitDestinationType,
 ///
 /// Example matches S, but not C, U or E.
 /// \code
-///   struct S {};
-///   class C {};
-///   union U {};
+///   struct S;
+///   class C;
+///   union U;
 ///   enum E {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{tagDecl(isStruct())} matches \match{struct S},
+/// but does not match \nomatch{class C}, \nomatch{union U} or
+/// \nomatch{enum E {}}.
 AST_MATCHER(TagDecl, isStruct) {
   return Node.isStruct();
 }
 
 /// Matches TagDecl object that are spelled with "union."
 ///
-/// Example matches U, but not C, S or E.
+/// Given
 /// \code
-///   struct S {};
-///   class C {};
-///   union U {};
+///   struct S;
+///   class C;
+///   union U;
 ///   enum E {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{tagDecl(isUnion())} matches \match{union U},
+/// but does not match \nomatch{struct S}, \nomatch{class C} or
+/// \nomatch{enum E {}}.
 AST_MATCHER(TagDecl, isUnion) {
   return Node.isUnion();
 }
 
 /// Matches TagDecl object that are spelled with "class."
 ///
-/// Example matches C, but not S, U or E.
-/// \code
-///   struct S {};
-///   class C {};
-///   union U {};
+/// Given
+/// \code
+///   struct S;
+///   class C;
+///   union U;
 ///   enum E {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{tagDecl(isClass())} matches \match{class C},
+/// but does not match \nomatch{struct S}, \nomatch{union U} or
+/// \nomatch{enum E {}}.
 AST_MATCHER(TagDecl, isClass) {
   return Node.isClass();
 }
 
 /// Matches TagDecl object that are spelled with "enum."
 ///
-/// Example matches E, but not C, S or U.
+/// Given
 /// \code
-///   struct S {};
-///   class C {};
-///   union U {};
+///   struct S;
+///   class C;
+///   union U;
 ///   enum E {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{tagDecl(isEnum())} matches \match{enum E {}},
+/// but does not match \nomatch{struct S}, \nomatch{class C} or
+/// \nomatch{union U}.
 AST_MATCHER(TagDecl, isEnum) {
   return Node.isEnum();
 }
@@ -6158,14 +8087,29 @@ AST_MATCHER(TagDecl, isEnum) {
 /// Matches the true branch expression of a conditional operator.
 ///
 /// Example 1 (conditional ternary operator): matches a
+/// Given
 /// \code
-///   condition ? a : b
+///   void foo(bool condition, int a, int b) {
+///     condition ? a : b;
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher
+/// \matcher{conditionalOperator(hasTrueExpression(expr().bind("true")))}
+/// matches \match{condition ? a : b},
+/// with \matcher{type=sub$expr()} matching \match{sub=true$a}.
 ///
 /// Example 2 (conditional binary operator): matches opaqueValueExpr(condition)
+/// Given
 /// \code
-///   condition ?: b
+///   void foo(bool condition, int a, int b) {
+///     condition ?: b;
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher \matcher{binaryConditionalOperator(hasTrueExpression(expr()))}
+/// matches \match{condition ?: b},
+/// with \matcher{type=sub$expr()} matching \match{sub=true$conditoin}.
 AST_MATCHER_P(AbstractConditionalOperator, hasTrueExpression,
               internal::Matcher<Expr>, InnerMatcher) {
   const Expr *Expression = Node.getTrueExpr();
@@ -6178,9 +8122,20 @@ AST_MATCHER_P(AbstractConditionalOperator, hasTrueExpression,
 ///
 /// Example matches b
 /// \code
-///   condition ? a : b
-///   condition ?: b
+///   void foo(bool condition, int a, int b) {
+///     condition ? a : b;
+///     condition ?: b;
+///   }
 /// \endcode
+/// \compile_args{-std=c++,c23-or-later}
+/// The matcher
+/// \matcher{conditionalOperator(hasFalseExpression(expr().bind("false")))}
+/// matches \match{condition ? a : b},
+/// with \matcher{type=sub$expr()} matching \match{sub=false$b}.
+/// The matcher
+/// \matcher{binaryConditionalOperator(hasFalseExpression(expr().bind("false")))}
+/// matches \match{condition ?: b},
+/// with \matcher{type=sub$expr()} matching \match{sub=false$b}.
 AST_MATCHER_P(AbstractConditionalOperator, hasFalseExpression,
               internal::Matcher<Expr>, InnerMatcher) {
   const Expr *Expression = Node.getFalseExpr();
@@ -6198,6 +8153,16 @@ AST_MATCHER_P(AbstractConditionalOperator, hasFalseExpression,
 ///   extern int vb;  // Doesn't match, as it doesn't define the variable.
 ///   void fa() {}
 ///   void fb();  // Doesn't match, as it has no body.
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{tagDecl(isDefinition())}
+/// matches \match{class A {}}.
+/// The matcher \matcher{varDecl(isDefinition())}
+/// matches \match{int va}.
+/// The matcher \matcher{functionDecl(isDefinition())}
+/// matches \match{void fa() {}}.
+///
+/// \code
 ///   @interface X
 ///   - (void)ma; // Doesn't match, interface is declaration.
 ///   @end
@@ -6205,6 +8170,9 @@ AST_MATCHER_P(AbstractConditionalOperator, hasFalseExpression,
 ///   - (void)ma {}
 ///   @end
 /// \endcode
+/// \compile_args{-ObjC}
+/// The matcher \matcher{objcMethodDecl(isDefinition())}
+/// matches \match{- (void)ma {}}
 ///
 /// Usable as: Matcher<TagDecl>, Matcher<VarDecl>, Matcher<FunctionDecl>,
 ///   Matcher<ObjCMethodDecl>
@@ -6225,6 +8193,12 @@ AST_POLYMORPHIC_MATCHER(isDefinition,
 ///   template <typename... Ts> void h(Ts...);
 ///   void i();
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{functionDecl(isVariadic())}
+/// matches \match{void f(...)},
+/// but does not match \nomatch{void g(int)},
+/// \nomatch{template <typename... Ts> void h(Ts...)},
+/// or \nomatch{void i()}.
 AST_MATCHER(FunctionDecl, isVariadic) {
   return Node.isVariadic();
 }
@@ -6236,16 +8210,17 @@ AST_MATCHER(FunctionDecl, isVariadic) {
 /// FIXME: What other kind of declarations would we need to generalize
 /// this to?
 ///
-/// Example matches A() in the last line
-///     (matcher = cxxConstructExpr(hasDeclaration(cxxMethodDecl(
-///         ofClass(hasName("A"))))))
+/// Given
 /// \code
 ///   class A {
 ///    public:
 ///     A();
+///     void foo();
 ///   };
-///   A a = A();
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(ofClass(hasName("A")))}
+/// matches \match{A()} and \match{void foo()}.
 AST_MATCHER_P(CXXMethodDecl, ofClass,
               internal::Matcher<CXXRecordDecl>, InnerMatcher) {
 
@@ -6265,10 +8240,14 @@ AST_MATCHER_P(CXXMethodDecl, ofClass,
 ///   class B : public A { void f(); };
 ///   class C : public B { void f(); };
 /// \endcode
-/// cxxMethodDecl(ofClass(hasName("C")),
-///               forEachOverridden(cxxMethodDecl().bind("b"))).bind("d")
-///   matches once, with "b" binding "A::f" and "d" binding "C::f" (Note
-///   that B::f is not overridden by C::f).
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(ofClass(hasName("C")),
+///               forEachOverridden(cxxMethodDecl().bind("b")))}
+/// matches \match{void f()} of \c C ,
+/// with \matcher{type=sub$cxxMethodDecl()} matching
+/// \match{sub=b$virtual void f()} of \c A ,
+/// but the matcher does not match \nomatch{void f()} of \c B because
+/// it is not overridden by C::f.
 ///
 /// The check can produce multiple matches in case of multiple inheritance, e.g.
 /// \code
@@ -6276,10 +8255,14 @@ AST_MATCHER_P(CXXMethodDecl, ofClass,
 ///   class A2 { virtual void f(); };
 ///   class C : public A1, public A2 { void f(); };
 /// \endcode
-/// cxxMethodDecl(ofClass(hasName("C")),
-///               forEachOverridden(cxxMethodDecl().bind("b"))).bind("d")
-///   matches twice, once with "b" binding "A1::f" and "d" binding "C::f", and
-///   once with "b" binding "A2::f" and "d" binding "C::f".
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(ofClass(hasName("C")),
+///               forEachOverridden(cxxMethodDecl().bind("b")))}
+/// matches \match{void f()} of \c C with the inner
+/// \matcher{type=sub$cxxMethodDecl()} matching \match{sub=b$virtual void f()}
+/// inside of \c A1 , and \match{void f()} of \c C with the inner
+/// \matcher{type=sub$cxxMethodDecl()} matching \match{sub=b$virtual void f()}
+/// inside of \c A2.
 AST_MATCHER_P(CXXMethodDecl, forEachOverridden,
               internal::Matcher<CXXMethodDecl>, InnerMatcher) {
   BoundNodesTreeBuilder Result;
@@ -6300,20 +8283,27 @@ AST_MATCHER_P(CXXMethodDecl, forEachOverridden,
 /// Matches declarations of virtual methods and C++ base specifers that specify
 /// virtual inheritance.
 ///
-/// Example:
+/// Given
 /// \code
 ///   class A {
 ///    public:
 ///     virtual void x(); // matches x
 ///   };
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(isVirtual())}
+/// matches \match{virtual void x()}.
 ///
-/// Example:
+/// Given
 /// \code
-///   class Base {};
-///   class DirectlyDerived : virtual Base {}; // matches Base
-///   class IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
+///   struct Base {};
+///   struct DirectlyDerived : virtual Base {}; // matches Base
+///   struct IndirectlyDerived : DirectlyDerived, Base {}; // matches Base
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{cxxRecordDecl(hasDirectBase(cxxBaseSpecifier(isVirtual())))}
+/// matches \match{struct DirectlyDerived : virtual Base {}}.
 ///
 /// Usable as: Matcher<CXXMethodDecl>, Matcher<CXXBaseSpecifier>
 AST_POLYMORPHIC_MATCHER(isVirtual,
@@ -6335,7 +8325,10 @@ AST_POLYMORPHIC_MATCHER(isVirtual,
 ///     void x();
 ///   };
 /// \endcode
-///   matches A::x but not B::x
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(isVirtualAsWritten())}
+/// matches \match{virtual void x()} of \c A,
+/// but does not match \notmatch{void x()} of \c B .
 AST_MATCHER(CXXMethodDecl, isVirtualAsWritten) {
   return Node.isVirtualAsWritten();
 }
@@ -6346,7 +8339,7 @@ AST_MATCHER(CXXConstructorDecl, isInheritingConstructor) {
 
 /// Matches if the given method or class declaration is final.
 ///
-/// Given:
+/// Given
 /// \code
 ///   class A final {};
 ///
@@ -6358,7 +8351,13 @@ AST_MATCHER(CXXConstructorDecl, isInheritingConstructor) {
 ///     void f() final;
 ///   };
 /// \endcode
-/// matches A and C::f, but not B, C, or B::f
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(isFinal())}
+/// matches \match{class A final {}},
+/// but does not match \nomatch{type=name$B} or \nomatch{type=name$C}.
+/// The matcher \matcher{cxxMethodDecl(isFinal())}
+/// matches \match{void f() final} in \c C ,
+/// but it does not match \nomatch{virtual void f()} in \c B .
 AST_POLYMORPHIC_MATCHER(isFinal,
                         AST_POLYMORPHIC_SUPPORTED_TYPES(CXXRecordDecl,
                                                         CXXMethodDecl)) {
@@ -6374,7 +8373,9 @@ AST_POLYMORPHIC_MATCHER(isFinal,
 ///     virtual void x() = 0;
 ///   };
 /// \endcode
-///   matches A::x
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(isPure())}
+/// matches \match{virtual void x() = 0}
 AST_MATCHER(CXXMethodDecl, isPure) { return Node.isPureVirtual(); }
 
 /// Matches if the given method declaration is const.
@@ -6386,8 +8387,10 @@ AST_MATCHER(CXXMethodDecl, isPure) { return Node.isPureVirtual(); }
 ///   void bar();
 /// };
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// cxxMethodDecl(isConst()) matches A::foo() but not A::bar()
+/// The matcher \matcher{cxxMethodDecl(isConst())}
+/// matches \match{void foo() const} but not \nomatch{void bar()}.
 AST_MATCHER(CXXMethodDecl, isConst) {
   return Node.isConst();
 }
@@ -6402,9 +8405,11 @@ AST_MATCHER(CXXMethodDecl, isConst) {
 ///   A &operator=(A &&);
 /// };
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// cxxMethodDecl(isCopyAssignmentOperator()) matches the first method but not
-/// the second one.
+/// The matcher \matcher{cxxMethodDecl(isCopyAssignmentOperator())}
+/// matches \match{A &operator=(const A &)}
+/// but does not match \nomatch{A &operator=(A &&)}
 AST_MATCHER(CXXMethodDecl, isCopyAssignmentOperator) {
   return Node.isCopyAssignmentOperator();
 }
@@ -6419,9 +8424,11 @@ AST_MATCHER(CXXMethodDecl, isCopyAssignmentOperator) {
 ///   A &operator=(A &&);
 /// };
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// cxxMethodDecl(isMoveAssignmentOperator()) matches the second method but not
-/// the first one.
+/// The matcher \matcher{cxxMethodDecl(isMoveAssignmentOperator())}
+/// matches \match{A &operator=(A &&)}
+/// but does not match \nomatch{A &operator=(const A &)}
 AST_MATCHER(CXXMethodDecl, isMoveAssignmentOperator) {
   return Node.isMoveAssignmentOperator();
 }
@@ -6436,10 +8443,12 @@ AST_MATCHER(CXXMethodDecl, isMoveAssignmentOperator) {
 ///   };
 ///   class B : public A {
 ///    public:
-///     virtual void x();
+///     void x() override;
 ///   };
 /// \endcode
-///   matches B::x
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxMethodDecl(isOverride())}
+///   matches \match{void x() override}
 AST_MATCHER(CXXMethodDecl, isOverride) {
   return Node.size_overridden_methods() > 0 || Node.hasAttr<OverrideAttr>();
 }
@@ -6454,7 +8463,10 @@ AST_MATCHER(CXXMethodDecl, isOverride) {
 ///     S(S &&) = delete; // #3
 ///   };
 /// \endcode
-/// cxxConstructorDecl(isUserProvided()) will match #1, but not #2 or #3.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(isUserProvided())}
+/// will match \match{S()}, but not \notmatch{S(const S &) = default} or
+/// \notmatch{S(S &&) = delete}
 AST_MATCHER(CXXMethodDecl, isUserProvided) {
   return Node.isUserProvided();
 }
@@ -6474,15 +8486,26 @@ AST_MATCHER(CXXMethodDecl, isUserProvided) {
 ///   };
 ///   template <class T>
 ///   class Z {
-///     void x() { this->m; }
+///     void x() {
+///       this->m;
+///       this->t;
+///       this->t->m;
+///     }
+///     int m;
+///     T* t;
 ///   };
 /// \endcode
-/// memberExpr(isArrow())
-///   matches this->x, x, y.x, a, this->b
-/// cxxDependentScopeMemberExpr(isArrow())
-///   matches this->m
-/// unresolvedMemberExpr(isArrow())
-///   matches this->f<T>, f<T>
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{memberExpr(isArrow())}
+/// matches \match{this->x}, \match{x}, \match{a},
+/// \match{this->b}, \match{this->m} and two times \match{count=2$this->t},
+/// once for the standalone member expression, and once for the member
+/// expression that later accesses \c m .
+/// Additionally, it does not match \nomatch{this->t->t}.
+/// The matcher \matcher{cxxDependentScopeMemberExpr(isArrow())}
+/// matches \match{this->t->m}, but not \nomatch{this->m} or \nomatch{this->t}.
+/// The matcher \matcher{unresolvedMemberExpr(isArrow())}
+/// matches \match{this->f<T>}, \match{f<T>}
 AST_POLYMORPHIC_MATCHER(
     isArrow, AST_POLYMORPHIC_SUPPORTED_TYPES(MemberExpr, UnresolvedMemberExpr,
                                              CXXDependentScopeMemberExpr)) {
@@ -6497,8 +8520,8 @@ AST_POLYMORPHIC_MATCHER(
 ///   void b(long);
 ///   void c(double);
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isInteger())))
-/// matches "a(int)", "b(long)", but not "c(double)".
+/// The matcher \matcher{functionDecl(hasAnyParameter(hasType(isInteger())))}
+/// \match{void a(int)}, \match{void b(long)}, but not \nomatch{void c(double)}.
 AST_MATCHER(QualType, isInteger) {
     return Node->isIntegerType();
 }
@@ -6511,8 +8534,10 @@ AST_MATCHER(QualType, isInteger) {
 ///   void b(unsigned long);
 ///   void c(double);
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isUnsignedInteger())))
-/// matches "b(unsigned long)", but not "a(int)" and "c(double)".
+/// The matcher
+/// \matcher{functionDecl(hasAnyParameter(hasType(isUnsignedInteger())))}
+/// matches \match{void b(unsigned long)},
+/// but it does not match \nomatch{void a(int)} and \nomatch{void c(double)}.
 AST_MATCHER(QualType, isUnsignedInteger) {
     return Node->isUnsignedIntegerType();
 }
@@ -6525,8 +8550,10 @@ AST_MATCHER(QualType, isUnsignedInteger) {
 ///   void b(unsigned long);
 ///   void c(double);
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isSignedInteger())))
-/// matches "a(int)", but not "b(unsigned long)" and "c(double)".
+/// The matcher
+/// \matcher{functionDecl(hasAnyParameter(hasType(isSignedInteger())))} matches
+/// \match{void a(int)}, but not \notmatch{void b(unsigned long)} or
+/// \notmatch{void c(double)}.
 AST_MATCHER(QualType, isSignedInteger) {
     return Node->isSignedIntegerType();
 }
@@ -6539,8 +8566,12 @@ AST_MATCHER(QualType, isSignedInteger) {
 ///   void b(wchar_t);
 ///   void c(double);
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isAnyCharacter())))
-/// matches "a(char)", "b(wchar_t)", but not "c(double)".
+/// \compile_args{-std=c++}
+///
+/// The matcher
+/// \matcher{functionDecl(hasAnyParameter(hasType(isAnyCharacter())))}
+/// \match{void a(char)}, \match{void b(wchar_t)}, but not
+/// \notmatch{void c(double)}.
 AST_MATCHER(QualType, isAnyCharacter) {
     return Node->isAnyCharacterType();
 }
@@ -6559,8 +8590,9 @@ AST_MATCHER(QualType, isAnyCharacter) {
 ///
 ///   int j;
 /// \endcode
-/// varDecl(hasType(isAnyPointer()))
-///   matches "int *i" and "Foo *f", but not "int j".
+/// \compile_args{-ObjC}
+/// The matcher \matcher{varDecl(hasType(isAnyPointer()))}
+/// \match{int *i} and \match{Foo *f}, but not \nomatch{int j}.
 AST_MATCHER(QualType, isAnyPointer) {
   return Node->isAnyPointerType();
 }
@@ -6574,12 +8606,12 @@ AST_MATCHER(QualType, isAnyPointer) {
 ///   void b(int const);
 ///   void c(const int);
 ///   void d(const int*);
-///   void e(int const) {};
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isConstQualified())))
-///   matches "void b(int const)", "void c(const int)" and
-///   "void e(int const) {}". It does not match d as there
-///   is no top-level const on the parameter type "const int *".
+/// The matcher
+/// \matcher{functionDecl(hasAnyParameter(hasType(isConstQualified())))}
+/// matches \match{void b(int const)} and \match{void c(const int)}.
+/// It does not match \notmatch{void d(const int*)} as there
+/// is no top-level \c const on the parameter type \c{const int *}.
 AST_MATCHER(QualType, isConstQualified) {
   return Node.isConstQualified();
 }
@@ -6593,12 +8625,12 @@ AST_MATCHER(QualType, isConstQualified) {
 ///   void b(int volatile);
 ///   void c(volatile int);
 ///   void d(volatile int*);
-///   void e(int volatile) {};
 /// \endcode
-/// functionDecl(hasAnyParameter(hasType(isVolatileQualified())))
-///   matches "void b(int volatile)", "void c(volatile int)" and
-///   "void e(int volatile) {}". It does not match d as there
-///   is no top-level volatile on the parameter type "volatile int *".
+/// The matcher
+/// \matcher{functionDecl(hasAnyParameter(hasType(isVolatileQualified())))}
+/// matches \match{void b(int volatile)} and \match{void c(volatile int)}.
+/// It does not match \notmatch{void d(volatile int*)} as there
+/// is no top-level volatile on the parameter type "volatile int *".
 AST_MATCHER(QualType, isVolatileQualified) {
   return Node.isVolatileQualified();
 }
@@ -6609,13 +8641,16 @@ AST_MATCHER(QualType, isVolatileQualified) {
 /// Given
 /// \code
 ///   typedef const int const_int;
-///   const_int i;
-///   int *const j;
+///   const_int i = 0;
+///   int *const j = nullptr;
 ///   int *volatile k;
 ///   int m;
 /// \endcode
-/// \c varDecl(hasType(hasLocalQualifiers())) matches only \c j and \c k.
-/// \c i is const-qualified but the qualifier is not local.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{varDecl(hasType(hasLocalQualifiers()))} matches
+/// \match{int *const j = nullptr} and \match{int *volatile k},
+///  bot not \notmatch{const_int i} because the const qualifier is not local.
 AST_MATCHER(QualType, hasLocalQualifiers) {
   return Node.hasLocalQualifiers();
 }
@@ -6625,13 +8660,15 @@ AST_MATCHER(QualType, hasLocalQualifiers) {
 ///
 /// Given
 /// \code
-///   struct { int first, second; } first, second;
-///   int i(second.first);
-///   int j(first.second);
+///   struct { int first = 0, second = 1; } first, second;
+///   int i = second.first;
+///   int j = first.second;
 /// \endcode
-/// memberExpr(member(hasName("first")))
-///   matches second.first
-///   but not first.second (because the member name there is "second").
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{memberExpr(member(hasName("first")))}
+/// matches \match{second.first}
+/// but not \notmatch{first.second}.
 AST_MATCHER_P(MemberExpr, member,
               internal::Matcher<ValueDecl>, InnerMatcher) {
   return InnerMatcher.matches(*Node.getMemberDecl(), Finder, Builder);
@@ -6648,11 +8685,14 @@ AST_MATCHER_P(MemberExpr, member,
 ///     int f(X x) { x.m; return m; }
 ///   };
 /// \endcode
-/// memberExpr(hasObjectExpression(hasType(cxxRecordDecl(hasName("X")))))
-///   matches `x.m`, but not `m`; however,
-/// memberExpr(hasObjectExpression(hasType(pointsTo(
-//      cxxRecordDecl(hasName("X"))))))
-///   matches `m` (aka. `this->m`), but not `x.m`.
+/// \compile_args{-std=c++}
+///
+/// The matcher
+/// \matcher{memberExpr(hasObjectExpression(hasType(cxxRecordDecl(hasName("X")))))}
+/// matches \match{x.m}, but not \nomatch{m}; however,
+/// The matcher \matcher{memberExpr(hasObjectExpression(hasType(pointsTo(
+/// cxxRecordDecl(hasName("X"))))))}
+/// matches \match{m} (aka. this->m), but not \nomatch{x.m}.
 AST_POLYMORPHIC_MATCHER_P(
     hasObjectExpression,
     AST_POLYMORPHIC_SUPPORTED_TYPES(MemberExpr, UnresolvedMemberExpr,
@@ -6674,8 +8714,9 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   namespace X { void b(); }
 ///   using X::b;
 /// \endcode
-/// usingDecl(hasAnyUsingShadowDecl(hasName("b"))))
-///   matches \code using X::b \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{usingDecl(hasAnyUsingShadowDecl(hasName("b")))}
+///   matches \match{using X::b}
 AST_MATCHER_P(BaseUsingDecl, hasAnyUsingShadowDecl,
               internal::Matcher<UsingShadowDecl>, InnerMatcher) {
   return matchesFirstInPointerRange(InnerMatcher, Node.shadow_begin(),
@@ -6692,9 +8733,11 @@ AST_MATCHER_P(BaseUsingDecl, hasAnyUsingShadowDecl,
 ///   using X::a;
 ///   using X::b;
 /// \endcode
-/// usingDecl(hasAnyUsingShadowDecl(hasTargetDecl(functionDecl())))
-///   matches \code using X::b \endcode
-///   but not \code using X::a \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{usingDecl(hasAnyUsingShadowDecl(hasTargetDecl(functionDecl())))}
+///   matches \match{using X::b}
+///   but not \notmatch{using X::a}
 AST_MATCHER_P(UsingShadowDecl, hasTargetDecl,
               internal::Matcher<NamedDecl>, InnerMatcher) {
   return InnerMatcher.matches(*Node.getTargetDecl(), Finder, Builder);
@@ -6705,26 +8748,44 @@ AST_MATCHER_P(UsingShadowDecl, hasTargetDecl,
 ///
 /// Given
 /// \code
-///   template <typename T> class X {}; class A {}; X<A> x;
+///   template <typename T> class X {};
+///   class A {};
+///   X<A> x;
 /// \endcode
-/// or
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("::X"),
+/// isTemplateInstantiation())}
+/// matches \match{type=typestr$class X<class A>}.
 /// \code
-///   template <typename T> class X {}; class A {}; template class X<A>;
+///   template <typename T> class X {};
+///   class A {};
+///   template class X<A>;
 /// \endcode
-/// or
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("::X"),
+/// isTemplateInstantiation())}
+/// matches \match{template class X<A>}
 /// \code
-///   template <typename T> class X {}; class A {}; extern template class X<A>;
+///   template <typename T> class X {};
+///   class A {};
+///   extern template class X<A>;
 /// \endcode
-/// cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
-///   matches the template instantiation of X<A>.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("::X"),
+/// isTemplateInstantiation())}
+/// matches \match{extern template class X<A>}
 ///
 /// But given
 /// \code
-///   template <typename T>  class X {}; class A {};
-///   template <> class X<A> {}; X<A> x;
+///   template <typename T>  class X {};
+///   class A {};
+///   template <> class X<A> {};
+///   X<A> x;
 /// \endcode
-/// cxxRecordDecl(hasName("::X"), isTemplateInstantiation())
-///   does not match, as X<A> is an explicit template specialization.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("::X"),
+/// isTemplateInstantiation())}
+///   \nomatch{} does not match, as X<A> is an explicit template specialization.
 ///
 /// Usable as: Matcher<FunctionDecl>, Matcher<VarDecl>, Matcher<CXXRecordDecl>
 AST_POLYMORPHIC_MATCHER(isTemplateInstantiation,
@@ -6743,11 +8804,15 @@ AST_POLYMORPHIC_MATCHER(isTemplateInstantiation,
 /// Given
 /// \code
 ///   template<typename T> void A(T t) { T i; }
-///   A(0);
-///   A(0U);
+///   void foo() {
+///     A(0);
+///     A(0U);
+///   }
 /// \endcode
-/// functionDecl(isInstantiated())
-///   matches 'A(int) {...};' and 'A(unsigned) {...}'.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{functionDecl(isInstantiated())}
+/// matches the two instantiations of \match{count=2$void A(T t) { T i; }} that
+/// are generated for \c int , and for \c{unsigned int}.
 AST_MATCHER_FUNCTION(internal::Matcher<Decl>, isInstantiated) {
   auto IsInstantiation = decl(anyOf(cxxRecordDecl(isTemplateInstantiation()),
                                     functionDecl(isTemplateInstantiation())));
@@ -6759,15 +8824,19 @@ AST_MATCHER_FUNCTION(internal::Matcher<Decl>, isInstantiated) {
 /// Given
 /// \code
 ///   int j;
-///   template<typename T> void A(T t) { T i; j += 42;}
-///   A(0);
-///   A(0U);
-/// \endcode
-/// declStmt(isInTemplateInstantiation())
-///   matches 'int i;' and 'unsigned i'.
-/// unless(stmt(isInTemplateInstantiation()))
-///   will NOT match j += 42; as it's shared between the template definition and
-///   instantiation.
+///   template<typename T> void A(T t) { T i; }
+///   void foo() {
+///     A(0);
+///     A(0U);
+///   }
+/// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{declStmt(isInTemplateInstantiation())}
+/// matches \match{count=2$T i;} twice, once for \c int and once for
+/// \c{unsigned int}.
+/// The matcher \matcher{declStmt(unless(isInTemplateInstantiation()))} will
+/// match \match{T i;} once inside the template definition, but not for any of
+/// the instantiated bodies.
 AST_MATCHER_FUNCTION(internal::Matcher<Stmt>, isInTemplateInstantiation) {
   return stmt(
       hasAncestor(decl(anyOf(cxxRecordDecl(isTemplateInstantiation()),
@@ -6782,8 +8851,9 @@ AST_MATCHER_FUNCTION(internal::Matcher<Stmt>, isInTemplateInstantiation) {
 ///   template<typename T> void A(T t) { }
 ///   template<> void A(int N) { }
 /// \endcode
-/// functionDecl(isExplicitTemplateSpecialization())
-///   matches the specialization A<int>().
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{functionDecl(isExplicitTemplateSpecialization())}
+///   matches the specialization \match{template<> void A(int N) { }}.
 ///
 /// Usable as: Matcher<FunctionDecl>, Matcher<VarDecl>, Matcher<CXXRecordDecl>
 AST_POLYMORPHIC_MATCHER(isExplicitTemplateSpecialization,
@@ -6794,6 +8864,13 @@ AST_POLYMORPHIC_MATCHER(isExplicitTemplateSpecialization,
 
 /// Matches \c TypeLocs for which the given inner
 /// QualType-matcher matches.
+///
+/// \code
+///   int a = 10;
+/// \endcode
+///
+/// The matcher \matcher{typeLoc(loc(qualType(isInteger())))}
+/// matches the \match{int} of \c a .
 AST_MATCHER_FUNCTION_P_OVERLOAD(internal::BindableMatcher<TypeLoc>, loc,
                                 internal::Matcher<QualType>, InnerMatcher, 0) {
   return internal::BindableMatcher<TypeLoc>(
@@ -6806,8 +8883,11 @@ AST_MATCHER_FUNCTION_P_OVERLOAD(internal::BindableMatcher<TypeLoc>, loc,
 /// \code
 ///   const int x = 0;
 /// \endcode
-/// qualifiedTypeLoc()
-///   matches `const int`.
+///
+/// The matcher \matcher{qualifiedTypeLoc()}
+/// matches the type of the variable declaration \c x . However, the
+/// current implementation of \c QualifiedTypeLoc does not store the source
+/// locations for the qualifiers of the type \match{int}.
 extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, QualifiedTypeLoc>
     qualifiedTypeLoc;
 
@@ -6816,11 +8896,14 @@ extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, QualifiedTypeLoc>
 ///
 /// Given
 /// \code
-///   int* const x;
-///   const int y;
+///   int* const x = nullptr;
+///   const int y = 0;
 /// \endcode
-/// qualifiedTypeLoc(hasUnqualifiedLoc(pointerTypeLoc()))
-///   matches the `TypeLoc` of the variable declaration of `x`, but not `y`.
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+///
+/// The matcher \matcher{qualifiedTypeLoc(hasUnqualifiedLoc(pointerTypeLoc()))}
+/// matches the type \match{int*} of the variable declaration \c{x}, but
+/// not \c{y}.
 AST_MATCHER_P(QualifiedTypeLoc, hasUnqualifiedLoc, internal::Matcher<TypeLoc>,
               InnerMatcher) {
   return InnerMatcher.matches(Node.getUnqualifiedLoc(), Finder, Builder);
@@ -6833,8 +8916,11 @@ AST_MATCHER_P(QualifiedTypeLoc, hasUnqualifiedLoc, internal::Matcher<TypeLoc>,
 ///   int f() { return 5; }
 ///   void g() {}
 /// \endcode
-/// functionDecl(hasReturnTypeLoc(loc(asString("int"))))
-///   matches the declaration of `f`, but not `g`.
+/// The matcher
+/// \matcher{functionDecl(hasReturnTypeLoc(typeLoc(loc(asString("int")))))}
+/// matches the declaration of \match{int f() { return 5; }} with
+/// \matcher{type=sub$typeLoc(loc(asString("int")))} matching the spelling of
+/// \match{sub=loc$int}, but the matcher does not match \notmatch{void g() {}}.
 AST_MATCHER_P(FunctionDecl, hasReturnTypeLoc, internal::Matcher<TypeLoc>,
               ReturnMatcher) {
   auto Loc = Node.getFunctionTypeLoc();
@@ -6847,8 +8933,8 @@ AST_MATCHER_P(FunctionDecl, hasReturnTypeLoc, internal::Matcher<TypeLoc>,
 /// \code
 ///   int* x;
 /// \endcode
-/// pointerTypeLoc()
-///   matches `int*`.
+/// The matcher \matcher{pointerTypeLoc()}
+///   matches \match{int*}.
 extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, PointerTypeLoc>
     pointerTypeLoc;
 
@@ -6859,8 +8945,8 @@ extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, PointerTypeLoc>
 /// \code
 ///   int* x;
 /// \endcode
-/// pointerTypeLoc(hasPointeeLoc(loc(asString("int"))))
-///   matches `int*`.
+/// The matcher \matcher{pointerTypeLoc(hasPointeeLoc(loc(asString("int"))))}
+///   matches \match{int*}.
 AST_MATCHER_P(PointerTypeLoc, hasPointeeLoc, internal::Matcher<TypeLoc>,
               PointeeMatcher) {
   return PointeeMatcher.matches(Node.getPointeeLoc(), Finder, Builder);
@@ -6874,8 +8960,10 @@ AST_MATCHER_P(PointerTypeLoc, hasPointeeLoc, internal::Matcher<TypeLoc>,
 ///   int& l = x;
 ///   int&& r = 3;
 /// \endcode
-/// referenceTypeLoc()
-///   matches `int&` and `int&&`.
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{referenceTypeLoc()}
+///   matches \match{int&} and \match{int&&}.
 extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, ReferenceTypeLoc>
     referenceTypeLoc;
 
@@ -6887,8 +8975,10 @@ extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, ReferenceTypeLoc>
 ///   int x = 3;
 ///   int& xx = x;
 /// \endcode
-/// referenceTypeLoc(hasReferentLoc(loc(asString("int"))))
-///   matches `int&`.
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{referenceTypeLoc(hasReferentLoc(loc(asString("int"))))}
+///   matches \match{int&}.
 AST_MATCHER_P(ReferenceTypeLoc, hasReferentLoc, internal::Matcher<TypeLoc>,
               ReferentMatcher) {
   return ReferentMatcher.matches(Node.getPointeeLoc(), Finder, Builder);
@@ -6901,8 +8991,11 @@ AST_MATCHER_P(ReferenceTypeLoc, hasReferentLoc, internal::Matcher<TypeLoc>,
 ///   template <typename T> class C {};
 ///   C<char> var;
 /// \endcode
-/// varDecl(hasTypeLoc(templateSpecializationTypeLoc(typeLoc())))
-///   matches `C<char> var`.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+/// templateSpecializationTypeLoc(typeLoc())))))}
+/// matches \match{C<char> var}.
 extern const internal::VariadicDynCastAllOfMatcher<
     TypeLoc, TemplateSpecializationTypeLoc>
     templateSpecializationTypeLoc;
@@ -6917,9 +9010,11 @@ extern const internal::VariadicDynCastAllOfMatcher<
 ///   template<typename T> class A {};
 ///   A<int> a;
 /// \endcode
-/// varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-///   hasTypeLoc(loc(asString("int")))))))
-///   matches `A<int> a`.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+/// templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
+/// hasTypeLoc(loc(asString("int")))))))))} matches \match{A<int> a}.
 AST_POLYMORPHIC_MATCHER_P(
     hasAnyTemplateArgumentLoc,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -6942,9 +9037,12 @@ AST_POLYMORPHIC_MATCHER_P(
 ///   A<double, int> b;
 ///   A<int, double> c;
 /// \endcode
-/// varDecl(hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
-///   hasTypeLoc(loc(asString("double")))))))
-///   matches `A<double, int> b`, but not `A<int, double> c`.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{varDecl(hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
+/// templateSpecializationTypeLoc(hasTemplateArgumentLoc(0,
+/// hasTypeLoc(loc(asString("double")))))))))}
+/// matches \match{A<double, int> b}, but not \notmatch{A<int, double> c}.
 AST_POLYMORPHIC_MATCHER_P2(
     hasTemplateArgumentLoc,
     AST_POLYMORPHIC_SUPPORTED_TYPES(ClassTemplateSpecializationDecl,
@@ -6963,8 +9061,8 @@ AST_POLYMORPHIC_MATCHER_P2(
 ///   struct s {};
 ///   struct s ss;
 /// \endcode
-/// elaboratedTypeLoc()
-///   matches the `TypeLoc` of the variable declaration of `ss`.
+/// The matcher \matcher{elaboratedTypeLoc()}
+/// matches the type \match{struct s} of \c ss.
 extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, ElaboratedTypeLoc>
     elaboratedTypeLoc;
 
@@ -6980,8 +9078,10 @@ extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, ElaboratedTypeLoc>
 ///   class D {};
 ///   class D d;
 /// \endcode
-/// elaboratedTypeLoc(hasNamedTypeLoc(templateSpecializationTypeLoc()));
-///   matches the `TypeLoc` of the variable declaration of `c`, but not `d`.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{elaboratedTypeLoc(hasNamedTypeLoc(templateSpecializationTypeLoc()))}
+///   matches \match{class C<int>}, but not \notmatch{ckass D}
 AST_MATCHER_P(ElaboratedTypeLoc, hasNamedTypeLoc, internal::Matcher<TypeLoc>,
               InnerMatcher) {
   return InnerMatcher.matches(Node.getNamedTypeLoc(), Finder, Builder);
@@ -6993,8 +9093,9 @@ AST_MATCHER_P(ElaboratedTypeLoc, hasNamedTypeLoc, internal::Matcher<TypeLoc>,
 /// \code
 ///  struct S { bool func(); };
 /// \endcode
-/// functionDecl(returns(booleanType()))
-///   matches "bool func();"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(returns(booleanType()))}
+/// \match{bool func()}.
 AST_MATCHER(Type, booleanType) {
   return Node.isBooleanType();
 }
@@ -7005,8 +9106,10 @@ AST_MATCHER(Type, booleanType) {
 /// \code
 ///  struct S { void func(); };
 /// \endcode
-/// functionDecl(returns(voidType()))
-///   matches "void func();"
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{functionDecl(returns(voidType()))}
+/// \match{void func()}.
 AST_MATCHER(Type, voidType) {
   return Node.isVoidType();
 }
@@ -7018,14 +9121,13 @@ using AstTypeMatcher = internal::VariadicDynCastAllOfMatcher<Type, NodeType>;
 ///
 /// Given
 /// \code
-///   struct A {};
-///   A a;
+///   enum E { Ok };
+///   enum E e;
 ///   int b;
 ///   float c;
-///   bool d;
 /// \endcode
-/// builtinType()
-///   matches "int b", "float c" and "bool d"
+/// The matcher \matcher{varDecl(hasType(builtinType()))}
+/// matches \match{int b} and \match{float c}.
 extern const AstTypeMatcher<BuiltinType> builtinType;
 
 /// Matches all kinds of arrays.
@@ -7036,8 +9138,9 @@ extern const AstTypeMatcher<BuiltinType> builtinType;
 ///   int b[4];
 ///   void f() { int c[a[0]]; }
 /// \endcode
-/// arrayType()
-///   matches "int a[]", "int b[4]" and "int c[a[0]]";
+/// The matcher \matcher{arrayType()}
+/// \match{type=typestr$int[4]}, \match{type=typestr$int[a[0]]} and
+/// \match{type=typestr$int[]};
 extern const AstTypeMatcher<ArrayType> arrayType;
 
 /// Matches C99 complex types.
@@ -7046,8 +9149,8 @@ extern const AstTypeMatcher<ArrayType> arrayType;
 /// \code
 ///   _Complex float f;
 /// \endcode
-/// complexType()
-///   matches "_Complex float f"
+/// The matcher \matcher{complexType()}
+/// \match{type=typestr$_Complex float}
 extern const AstTypeMatcher<ComplexType> complexType;
 
 /// Matches any real floating-point type (float, double, long double).
@@ -7057,8 +9160,9 @@ extern const AstTypeMatcher<ComplexType> complexType;
 ///   int i;
 ///   float f;
 /// \endcode
-/// realFloatingPointType()
-///   matches "float f" but not "int i"
+/// The matcher \matcher{type(realFloatingPointType())}
+/// matches \match{type=typestr$float}
+/// but does not match \nomatch{type=typestr$int}.
 AST_MATCHER(Type, realFloatingPointType) {
   return Node.isRealFloatingType();
 }
@@ -7072,8 +9176,10 @@ AST_MATCHER(Type, realFloatingPointType) {
 ///   A a[7];
 ///   int b[7];
 /// \endcode
-/// arrayType(hasElementType(builtinType()))
-///   matches "int b[7]"
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{arrayType(hasElementType(builtinType()))}
+/// \match{type=typestr$int[7]}
 ///
 /// Usable as: Matcher<ArrayType>, Matcher<ComplexType>
 AST_TYPELOC_TRAVERSE_MATCHER_DECL(hasElementType, getElement,
@@ -7084,14 +9190,14 @@ AST_TYPELOC_TRAVERSE_MATCHER_DECL(hasElementType, getElement,
 ///
 /// Given
 /// \code
-///   void() {
+///   void foo() {
 ///     int a[2];
 ///     int b[] = { 2, 3 };
 ///     int c[b[0]];
 ///   }
 /// \endcode
-/// constantArrayType()
-///   matches "int a[2]"
+/// The matcher \matcher{constantArrayType()}
+/// \match{type=typestr$int[2]}
 extern const AstTypeMatcher<ConstantArrayType> constantArrayType;
 
 /// Matches nodes that have the specified size.
@@ -7105,10 +9211,11 @@ extern const AstTypeMatcher<ConstantArrayType> constantArrayType;
 ///   wchar_t *ws = L"abcd";
 ///   char *w = "a";
 /// \endcode
-/// constantArrayType(hasSize(42))
-///   matches "int a[42]" and "int b[2 * 21]"
-/// stringLiteral(hasSize(4))
-///   matches "abcd", L"abcd"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{constantArrayType(hasSize(42))}
+/// matches \match{type=typestr;count=2$int[42]} twice.
+/// The matcher \matcher{stringLiteral(hasSize(4))}
+/// matches \match{"abcd"} and \match{L"abcd"}.
 AST_POLYMORPHIC_MATCHER_P(hasSize,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(ConstantArrayType,
                                                           StringLiteral),
@@ -7125,8 +9232,9 @@ AST_POLYMORPHIC_MATCHER_P(hasSize,
 ///     T data[Size];
 ///   };
 /// \endcode
-/// dependentSizedArrayType()
-///   matches "T data[Size]"
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{dependentSizedArrayType()}
+/// \match{type=typestr$T[Size]}
 extern const AstTypeMatcher<DependentSizedArrayType> dependentSizedArrayType;
 
 /// Matches C++ extended vector type where either the type or size is
@@ -7139,8 +9247,9 @@ extern const AstTypeMatcher<DependentSizedArrayType> dependentSizedArrayType;
 ///     typedef T __attribute__((ext_vector_type(Size))) type;
 ///   };
 /// \endcode
-/// dependentSizedExtVectorType()
-///   matches "T __attribute__((ext_vector_type(Size)))"
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{dependentSizedExtVectorType()}
+/// \match{type=typestr$T __attribute__((ext_vector_type(Size)))}
 extern const AstTypeMatcher<DependentSizedExtVectorType>
     dependentSizedExtVectorType;
 
@@ -7152,8 +9261,8 @@ extern const AstTypeMatcher<DependentSizedExtVectorType>
 ///   int b[42];
 ///   void f(int c[]) { int d[a[0]]; };
 /// \endcode
-/// incompleteArrayType()
-///   matches "int a[]" and "int c[]"
+/// The matcher \matcher{incompleteArrayType()}
+/// \match{type=typestr$int[]} and \match{type=typestr$int[]}
 extern const AstTypeMatcher<IncompleteArrayType> incompleteArrayType;
 
 /// Matches C arrays with a specified size that is not an
@@ -7162,13 +9271,13 @@ extern const AstTypeMatcher<IncompleteArrayType> incompleteArrayType;
 /// Given
 /// \code
 ///   void f() {
-///     int a[] = { 2, 3 }
+///     int a[] = { 2, 3 };
 ///     int b[42];
 ///     int c[a[0]];
 ///   }
 /// \endcode
-/// variableArrayType()
-///   matches "int c[a[0]]"
+/// The matcher \matcher{variableArrayType()}
+/// \match{type=typestr$int[a[0]]}
 extern const AstTypeMatcher<VariableArrayType> variableArrayType;
 
 /// Matches \c VariableArrayType nodes that have a specific size
@@ -7180,9 +9289,10 @@ extern const AstTypeMatcher<VariableArrayType> variableArrayType;
 ///     int a[b];
 ///   }
 /// \endcode
-/// variableArrayType(hasSizeExpr(ignoringImpCasts(declRefExpr(to(
-///   varDecl(hasName("b")))))))
-///   matches "int a[b]"
+/// The matcher
+/// \matcher{variableArrayType(hasSizeExpr(ignoringImpCasts(declRefExpr(to(
+///   varDecl(hasName("b")))))))}
+/// matches \match{type=typestr$int[b]}
 AST_MATCHER_P(VariableArrayType, hasSizeExpr,
               internal::Matcher<Expr>, InnerMatcher) {
   return InnerMatcher.matches(*Node.getSizeExpr(), Finder, Builder);
@@ -7194,8 +9304,8 @@ AST_MATCHER_P(VariableArrayType, hasSizeExpr,
 /// \code
 ///   _Atomic(int) i;
 /// \endcode
-/// atomicType()
-///   matches "_Atomic(int) i"
+/// The matcher \matcher{atomicType()}
+/// \match{type=typestr$_Atomic(int)}
 extern const AstTypeMatcher<AtomicType> atomicType;
 
 /// Matches atomic types with a specific value type.
@@ -7205,8 +9315,8 @@ extern const AstTypeMatcher<AtomicType> atomicType;
 ///   _Atomic(int) i;
 ///   _Atomic(float) f;
 /// \endcode
-/// atomicType(hasValueType(isInteger()))
-///  matches "_Atomic(int) i"
+/// The matcher \matcher{atomicType(hasValueType(isInteger()))}
+/// \match{type=typestr$_Atomic(int)}.
 ///
 /// Usable as: Matcher<AtomicType>
 AST_TYPELOC_TRAVERSE_MATCHER_DECL(hasValueType, getValue,
@@ -7214,26 +9324,32 @@ AST_TYPELOC_TRAVERSE_MATCHER_DECL(hasValueType, getValue,
 
 /// Matches types nodes representing C++11 auto types.
 ///
-/// Given:
+/// Given
 /// \code
-///   auto n = 4;
-///   int v[] = { 2, 3 }
-///   for (auto i : v) { }
+///   void foo() {
+///     auto n = 4;
+///     int v[] = { 2, 3 };
+///     for (auto i : v) { };
+///   }
 /// \endcode
-/// autoType()
-///   matches "auto n" and "auto i"
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{autoType()}
+/// matches the \match{type=typestr;count=5$auto} of \c n and \c i ,
+/// as well as the auto types for the implicitly generated code of the range-for
+/// loop (for the range, the begin iterator and the end iterator).
 extern const AstTypeMatcher<AutoType> autoType;
 
 /// Matches types nodes representing C++11 decltype(<expr>) types.
 ///
-/// Given:
+/// Given
 /// \code
 ///   short i = 1;
 ///   int j = 42;
 ///   decltype(i + j) result = i + j;
 /// \endcode
-/// decltypeType()
-///   matches "decltype(i + j)"
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{decltypeType()}
+/// \match{type=typestr$decltype(i + j)}
 extern const AstTypeMatcher<DecltypeType> decltypeType;
 
 /// Matches \c AutoType nodes where the deduced type is a specific type.
@@ -7246,8 +9362,10 @@ extern const AstTypeMatcher<DecltypeType> decltypeType;
 ///   auto a = 1;
 ///   auto b = 2.0;
 /// \endcode
-/// autoType(hasDeducedType(isInteger()))
-///   matches "auto a"
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+/// \matcher{varDecl(hasType(autoType(hasDeducedType(isInteger()))))}
+/// matches \match{auto a = 1}, but does not match \nomatch{auto b = 2.0}.
 ///
 /// Usable as: Matcher<AutoType>
 AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
@@ -7260,8 +9378,11 @@ AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
 ///   decltype(1) a = 1;
 ///   decltype(2.0) b = 2.0;
 /// \endcode
-/// decltypeType(hasUnderlyingType(isInteger()))
-///   matches the type of "a"
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{decltypeType(hasUnderlyingType(isInteger()))}
+/// matches the type \match{type=typestr$decltype(1)} of the variable
+/// declaration of \c a .
 ///
 /// Usable as: Matcher<DecltypeType>, Matcher<UsingType>
 AST_TYPE_TRAVERSE_MATCHER(hasUnderlyingType, getUnderlyingType,
@@ -7275,8 +9396,12 @@ AST_TYPE_TRAVERSE_MATCHER(hasUnderlyingType, getUnderlyingType,
 ///   int (*f)(int);
 ///   void g();
 /// \endcode
-/// functionType()
-///   matches "int (*f)(int)" and the type of "g".
+/// The matcher \matcher{functionType()}
+/// \match{type=typestr$int (int)} and the type of
+/// \match{std=c++,c23-or-later;type=typestr$void (void)} in C++ and in C23 and
+/// later. Before C23, the function type for \c f will be matched the same way,
+/// but the function type for \c g will match
+/// \match{std=c17-or-earlier;type=typestr$void ()}.
 extern const AstTypeMatcher<FunctionType> functionType;
 
 /// Matches \c FunctionProtoType nodes.
@@ -7286,9 +9411,11 @@ extern const AstTypeMatcher<FunctionType> functionType;
 ///   int (*f)(int);
 ///   void g();
 /// \endcode
-/// functionProtoType()
-///   matches "int (*f)(int)" and the type of "g" in C++ mode.
-///   In C mode, "g" is not matched because it does not contain a prototype.
+/// The matcher \matcher{functionProtoType()}
+/// matches the type \match{type=typestr$int (int)} of 'f' and the type
+/// \match{std=c++,c23-or-later;type=typestr$void (void)} of 'g' in C++ mode.
+/// In C, the type \nomatch{std=c;type=typestr$void ()} of 'g' is not
+/// matched because it does not contain a prototype.
 extern const AstTypeMatcher<FunctionProtoType> functionProtoType;
 
 /// Matches \c ParenType nodes.
@@ -7299,8 +9426,9 @@ extern const AstTypeMatcher<FunctionProtoType> functionProtoType;
 ///   int *array_of_ptrs[4];
 /// \endcode
 ///
-/// \c varDecl(hasType(pointsTo(parenType()))) matches \c ptr_to_array but not
-/// \c array_of_ptrs.
+/// The matcher \matcher{varDecl(hasType(pointsTo(parenType())))}
+/// matches \match{int (*ptr_to_array)[4]}, but not
+/// \nomatch{int *array_of_ptrs[4]}.
 extern const AstTypeMatcher<ParenType> parenType;
 
 /// Matches \c ParenType nodes where the inner type is a specific type.
@@ -7311,8 +9439,10 @@ extern const AstTypeMatcher<ParenType> parenType;
 ///   int (*ptr_to_func)(int);
 /// \endcode
 ///
-/// \c varDecl(hasType(pointsTo(parenType(innerType(functionType()))))) matches
-/// \c ptr_to_func but not \c ptr_to_array.
+/// The matcher
+/// \matcher{varDecl(hasType(pointsTo(parenType(innerType(functionType())))))}
+/// matches \match{int (*ptr_to_func)(int)} but not
+/// \nomatch{int (*ptr_to_array)[4]}.
 ///
 /// Usable as: Matcher<ParenType>
 AST_TYPE_TRAVERSE_MATCHER(innerType, getInnerType,
@@ -7327,11 +9457,12 @@ extern const AstTypeMatcher<BlockPointerType> blockPointerType;
 /// Matches member pointer types.
 /// Given
 /// \code
-///   struct A { int i; }
-///   A::* ptr = A::i;
+///   struct A { int i; };
+///   int A::* ptr = &A::i;
 /// \endcode
-/// memberPointerType()
-///   matches "A::* ptr"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{memberPointerType()}
+/// matches \match{type=typestr$int struct A::*}.
 extern const AstTypeMatcher<MemberPointerType> memberPointerType;
 
 /// Matches pointer types, but does not match Objective-C object pointer
@@ -7339,16 +9470,24 @@ extern const AstTypeMatcher<MemberPointerType> memberPointerType;
 ///
 /// Given
 /// \code
-///   int *a;
-///   int &b = *a;
-///   int c = 5;
+///   typedef int* int_ptr;
+///   void foo(char *str,
+///            int val,
+///            int *val_ptr,
+///            int_ptr not_a_ptr,
+///            int_ptr *ptr);
+/// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{parmVarDecl(hasType(pointerType()))}
+/// matches \match{char *str}, \match{int *val_ptr} and
+/// \match{int_ptr *ptr}.
 ///
+/// \code
 ///   @interface Foo
 ///   @end
 ///   Foo *f;
 /// \endcode
-/// pointerType()
-///   matches "int *a", but does not match "Foo *f".
+/// \compile_args{-ObjC}
 extern const AstTypeMatcher<PointerType> pointerType;
 
 /// Matches an Objective-C object pointer type, which is different from
@@ -7362,8 +9501,10 @@ extern const AstTypeMatcher<PointerType> pointerType;
 ///   @end
 ///   Foo *f;
 /// \endcode
-/// pointerType()
-///   matches "Foo *f", but does not match "int *a".
+/// \compile_args{-ObjC}
+/// The matcher \matcher{pointerType()}
+/// matches \match{type=typestr$Foo *}, but does not match
+/// \nomatch{type=typestr$int *}.
 extern const AstTypeMatcher<ObjCObjectPointerType> objcObjectPointerType;
 
 /// Matches both lvalue and rvalue reference types.
@@ -7378,13 +9519,18 @@ extern const AstTypeMatcher<ObjCObjectPointerType> objcObjectPointerType;
 ///   auto &&f = 2;
 ///   int g = 5;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c referenceType() matches the types of \c b, \c c, \c d, \c e, and \c f.
+/// The matcher \matcher{referenceType()} matches the type
+/// \match{type=typestr$int &} of \c b , the type \match{type=typestr$int &&} of
+/// \c c, the type
+/// \match{type=typestr$auto &} \c d, and the type
+/// \match{type=typestr;count=2$auto &&} of \c e and \c f.
 extern const AstTypeMatcher<ReferenceType> referenceType;
 
 /// Matches lvalue reference types.
 ///
-/// Given:
+/// Given
 /// \code
 ///   int *a;
 ///   int &b = *a;
@@ -7394,14 +9540,17 @@ extern const AstTypeMatcher<ReferenceType> referenceType;
 ///   auto &&f = 2;
 ///   int g = 5;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c lValueReferenceType() matches the types of \c b, \c d, and \c e. \c e is
-/// matched since the type is deduced as int& by reference collapsing rules.
+/// The matcher \matcher{lValueReferenceType()} matches the type
+/// \match{type=typestr$int &} of \c b and the type \match{type=typestr$auto &}
+/// of \c d.
+/// FIXME: figure out why auto changechange matches twice
 extern const AstTypeMatcher<LValueReferenceType> lValueReferenceType;
 
 /// Matches rvalue reference types.
 ///
-/// Given:
+/// Given
 /// \code
 ///   int *a;
 ///   int &b = *a;
@@ -7411,9 +9560,11 @@ extern const AstTypeMatcher<LValueReferenceType> lValueReferenceType;
 ///   auto &&f = 2;
 ///   int g = 5;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c rValueReferenceType() matches the types of \c c and \c f. \c e is not
-/// matched as it is deduced to int& by reference collapsing rules.
+/// The matcher \matcher{rValueReferenceType()} matches the type
+/// \match{type=typestr$int &&} of \c c and the type
+/// \match{type=typestr;count=2$auto &&} of \c e and \c f.
 extern const AstTypeMatcher<RValueReferenceType> rValueReferenceType;
 
 /// Narrows PointerType (and similar) matchers to those where the
@@ -7422,11 +9573,15 @@ extern const AstTypeMatcher<RValueReferenceType> rValueReferenceType;
 /// Given
 /// \code
 ///   int *a;
-///   int const *b;
-///   float const *f;
+///   const int *b;
+///   int * const c = nullptr;
+///   const float *f;
 /// \endcode
-/// pointerType(pointee(isConstQualified(), isInteger()))
-///   matches "int const *b"
+/// \compile_args{-std=c++11-or-later,c23-or-later}
+/// The matcher \matcher{pointerType(pointee(isConstQualified(), isInteger()))}
+/// matches \match{type=typestr$const int *},
+/// but does not match \nomatch{type=typestr$int * const}
+/// or \nomatch{type=typestr$const float *}.
 ///
 /// Usable as: Matcher<BlockPointerType>, Matcher<MemberPointerType>,
 ///   Matcher<PointerType>, Matcher<ReferenceType>
@@ -7440,9 +9595,10 @@ AST_TYPELOC_TRAVERSE_MATCHER_DECL(
 /// Given
 /// \code
 ///   typedef int X;
+///   X x = 0;
 /// \endcode
-/// typedefType()
-///   matches "typedef int X"
+/// The matcher \matcher{typedefType()}
+/// matches \match{type=typestr$X}.
 extern const AstTypeMatcher<TypedefType> typedefType;
 
 /// Matches qualified types when the qualifier is applied via a macro.
@@ -7453,8 +9609,12 @@ extern const AstTypeMatcher<TypedefType> typedefType;
 ///   typedef void (CDECL *X)();
 ///   typedef void (__attribute__((cdecl)) *Y)();
 /// \endcode
-/// macroQualifiedType()
-///   matches the type of the typedef declaration of \c X but not \c Y.
+/// The matcher \matcher{macroQualifiedType()}
+/// matches the type \match{type=typestr;std=c++,c23-or-later$CDECL void
+/// (void)} of the typedef declaration of \c X , unless when in C98-C17, there
+/// \match{type=typestr;std=c17-or-earlier$CDECL void ()},
+/// but it does not match the type
+/// \nomatch{type=typestr$__attribute((cdecl)) void ()} of \c Y .
 extern const AstTypeMatcher<MacroQualifiedType> macroQualifiedType;
 
 /// Matches enum types.
@@ -7467,9 +9627,11 @@ extern const AstTypeMatcher<MacroQualifiedType> macroQualifiedType;
 ///   C c;
 ///   S s;
 /// \endcode
-//
-/// \c enumType() matches the type of the variable declarations of both \c c and
-/// \c s.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{enumType()} matches the type
+/// \match{type=typestr$enum C} of \c c ,
+/// and the type \match{type=typestr$enum S} of \c s .
 extern const AstTypeMatcher<EnumType> enumType;
 
 /// Matches template specialization types.
@@ -7479,12 +9641,16 @@ extern const AstTypeMatcher<EnumType> enumType;
 ///   template <typename T>
 ///   class C { };
 ///
-///   template class C<int>;  // A
-///   C<char> var;            // B
+///   template class C<int>;
+///   C<int> intvar;
+///   C<char> charvar;
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
 ///
-/// \c templateSpecializationType() matches the type of the explicit
-/// instantiation in \c A and the type of the variable declaration in \c B.
+/// The matcher \matcher{templateSpecializationType()} matches the type
+/// \match{type=typestr$C<int>} of the explicit instantiation in \c A and the
+/// type \match{type=typestr$C<char>} of the variable declaration in
+/// \c B.
 extern const AstTypeMatcher<TemplateSpecializationType>
     templateSpecializationType;
 
@@ -7498,19 +9664,23 @@ extern const AstTypeMatcher<TemplateSpecializationType>
 ///
 ///   C c(123);
 /// \endcode
-/// \c deducedTemplateSpecializationType() matches the type in the declaration
-/// of the variable \c c.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++17-or-later}
+/// The matcher \matcher{deducedTemplateSpecializationType()} matches the type
+/// \match{type=typestr$C} of the declaration of the variable \c c.
 extern const AstTypeMatcher<DeducedTemplateSpecializationType>
     deducedTemplateSpecializationType;
 
 /// Matches types nodes representing unary type transformations.
 ///
-/// Given:
+/// Given
 /// \code
-///   typedef __underlying_type(T) type;
+///   template <typename T> struct A {
+///     typedef __underlying_type(T) type;
+///   };
 /// \endcode
-/// unaryTransformType()
-///   matches "__underlying_type(T)"
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{unaryTransformType()}
+/// matches \match{type=typestr$__underlying_type(T)}
 extern const AstTypeMatcher<UnaryTransformType> unaryTransformType;
 
 /// Matches record types (e.g. structs, classes).
@@ -7523,24 +9693,34 @@ extern const AstTypeMatcher<UnaryTransformType> unaryTransformType;
 ///   C c;
 ///   S s;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c recordType() matches the type of the variable declarations of both \c c
-/// and \c s.
+/// The matcher \matcher{recordType()} matches the type
+/// \match{type=typestr;count=3$class C} of the variable declaration of \c c and
+/// matches the type \match{type=typestr;count=3$struct S} of the variable
+/// declaration of \c s.
+/// Both of these types are matched three times, once for the type of the
+/// variable, once for the definition of the class, and once for the type of the
+/// implicit class declaration.
 extern const AstTypeMatcher<RecordType> recordType;
 
 /// Matches tag types (record and enum types).
 ///
 /// Given
 /// \code
-///   enum E {};
+///   enum E { Ok };
 ///   class C {};
 ///
 ///   E e;
 ///   C c;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c tagType() matches the type of the variable declarations of both \c e
-/// and \c c.
+/// The matcher \matcher{tagType()} matches the type
+/// \match{type=typestr$enum E} of variable \c e and the type
+/// \match{type=typestr;count=3;std=c++$class C} three times, once for the type
+/// of the variable \c c , once for the type of the class definition and once of
+/// the type in the implicit class declaration.
 extern const AstTypeMatcher<TagType> tagType;
 
 /// Matches types specified with an elaborated type keyword or with a
@@ -7555,16 +9735,22 @@ extern const AstTypeMatcher<TagType> tagType;
 ///   }
 ///   class C {};
 ///
-///   class C c;
+///   C c;
 ///   N::M::D d;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c elaboratedType() matches the type of the variable declarations of both
-/// \c c and \c d.
+/// The matcher \matcher{elaboratedType()} matches the type
+/// \match{type=typestr;count=3$C} three times. Once for the type of the
+/// variable \c c, once for the type of the class definition and once for the
+/// type in the implicit class declaration. For \c{class D}, it matches
+/// \match{type=typestr$N::M::D} of variable \c d and its class definition and
+/// implicit class declaration \match{type=typestr;count=2$D} one time
+/// respectively.
 extern const AstTypeMatcher<ElaboratedType> elaboratedType;
 
 /// Matches ElaboratedTypes whose qualifier, a NestedNameSpecifier,
-/// matches \c InnerMatcher if the qualifier exists.
+///   matches \c InnerMatcher if the qualifier exists.
 ///
 /// Given
 /// \code
@@ -7575,9 +9761,12 @@ extern const AstTypeMatcher<ElaboratedType> elaboratedType;
 ///   }
 ///   N::M::D d;
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// \c elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N"))))
-/// matches the type of the variable declaration of \c d.
+/// The matcher
+/// \matcher{elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))}
+///   matches the type \match{type=typestr$N::M::D} of the variable declaration
+///   of \c d.
 AST_MATCHER_P(ElaboratedType, hasQualifier,
               internal::Matcher<NestedNameSpecifier>, InnerMatcher) {
   if (const NestedNameSpecifier *Qualifier = Node.getQualifier())
@@ -7592,15 +9781,15 @@ AST_MATCHER_P(ElaboratedType, hasQualifier,
 /// \code
 ///   namespace N {
 ///     namespace M {
-///       class D {};
+///       enum E { Ok };
 ///     }
 ///   }
-///   N::M::D d;
+///   N::M::E e = N::M::Ok;
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// \c elaboratedType(namesType(recordType(
-/// hasDeclaration(namedDecl(hasName("D")))))) matches the type of the variable
-/// declaration of \c d.
+/// The matcher \matcher{elaboratedType(namesType(enumType()))}
+/// matches the type \match{type=typestr$N::M::E} of the declaration of \c e .
 AST_MATCHER_P(ElaboratedType, namesType, internal::Matcher<QualType>,
               InnerMatcher) {
   return InnerMatcher.matches(Node.getNamedType(), Finder, Builder);
@@ -7614,8 +9803,10 @@ AST_MATCHER_P(ElaboratedType, namesType, internal::Matcher<QualType>,
 ///   using a::S;
 ///   S s;
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// \c usingType() matches the type of the variable declaration of \c s.
+/// The matcher \matcher{usingType()} matches the type \match{type=typestr$a::S}
+/// of the variable declaration of \c s.
 extern const AstTypeMatcher<UsingType> usingType;
 
 /// Matches types that represent the result of substituting a type for a
@@ -7625,11 +9816,18 @@ extern const AstTypeMatcher<UsingType> usingType;
 /// \code
 ///   template <typename T>
 ///   void F(T t) {
+///     T local;
 ///     int i = 1 + t;
 ///   }
+///   void f() {
+///     F(0);
+///   }
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
 ///
-/// \c substTemplateTypeParmType() matches the type of 't' but not '1'
+/// The matcher \matcher{varDecl(hasType(substTemplateTypeParmType()))}
+/// matches \match{T t} and \match{T local} for the substituted template type
+/// \c int in the instantiation of \c F .
 extern const AstTypeMatcher<SubstTemplateTypeParmType>
     substTemplateTypeParmType;
 
@@ -7643,46 +9841,67 @@ extern const AstTypeMatcher<SubstTemplateTypeParmType>
 ///   int i;
 ///   double j = F(i);
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
 ///
-/// \c substTemplateTypeParmType(hasReplacementType(type())) matches int
+/// The matcher \matcher{substTemplateTypeParmType(hasReplacementType(type()))}
+/// matches \match{type=typestr$int}.
 AST_TYPE_TRAVERSE_MATCHER(
     hasReplacementType, getReplacementType,
     AST_POLYMORPHIC_SUPPORTED_TYPES(SubstTemplateTypeParmType));
 
 /// Matches template type parameter types.
 ///
-/// Example matches T, but not int.
-///     (matcher = templateTypeParmType())
+/// Given
 /// \code
 ///   template <typename T> void f(int i);
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher \matcher{templateTypeParmType()} matches \match{type=typestr$T},
+/// but does not match \nomatch{type=typestr$int}.
 extern const AstTypeMatcher<TemplateTypeParmType> templateTypeParmType;
 
 /// Matches injected class name types.
 ///
-/// Example matches S s, but not S<T> s.
-///     (matcher = parmVarDecl(hasType(injectedClassNameType())))
+/// Given
 /// \code
 ///   template <typename T> struct S {
 ///     void f(S s);
 ///     void g(S<T> s);
 ///   };
 /// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++}
+/// The matcher
+/// \matcher{parmVarDecl(hasType(elaboratedType(namesType(injectedClassNameType()))))}
+/// matches \match{S s}, but not \notmatch{S<T> s}
 extern const AstTypeMatcher<InjectedClassNameType> injectedClassNameType;
 
 /// Matches decayed type
-/// Example matches i[] in declaration of f.
-///     (matcher = valueDecl(hasType(decayedType(hasDecayedType(pointerType())))))
-/// Example matches i[1].
-///     (matcher = expr(hasType(decayedType(hasDecayedType(pointerType())))))
 /// \code
 ///   void f(int i[]) {
 ///     i[1] = 0;
 ///   }
 /// \endcode
+/// The matcher
+/// \matcher{valueDecl(hasType(decayedType(hasDecayedType(pointerType()))))}
+/// matches \match{int i[]} in declaration of \c{f}.
+/// The matcher
+/// \matcher{expr(hasType(decayedType(hasDecayedType(pointerType()))))}
+/// matches \match{count=2$i} twice, once for the \c DeclRefExpr and oncde for
+/// the cast from an l- to an r-value in \c{i[1]}.
+///
 extern const AstTypeMatcher<DecayedType> decayedType;
 
 /// Matches the decayed type, whoes decayed type matches \c InnerMatcher
+///
+/// Given
+/// \code
+///   void f(int i[]) {
+///     i[1] = 0;
+///   }
+/// \endcode
+///
+/// The matcher \matcher{parmVarDecl(hasType(decayedType()))}
+/// matches \match{int i[]}.
 AST_MATCHER_P(DecayedType, hasDecayedType, internal::Matcher<QualType>,
               InnerType) {
   return InnerType.matches(Node.getDecayedType(), Finder, Builder);
@@ -7699,9 +9918,10 @@ AST_MATCHER_P(DecayedType, hasDecayedType, internal::Matcher<QualType>,
 ///     }
 ///   }
 /// \endcode
+/// \compile_args{-std=c++}
 ///
-/// \c cxxRcordDecl(hasDeclContext(namedDecl(hasName("M")))) matches the
-/// declaration of \c class \c D.
+/// The matcher \matcher{cxxRecordDecl(hasDeclContext(namedDecl(hasName("M"))))}
+/// matches the definition of \match{class D {}}.
 AST_MATCHER_P(Decl, hasDeclContext, internal::Matcher<Decl>, InnerMatcher) {
   const DeclContext *DC = Node.getDeclContext();
   if (!DC) return false;
@@ -7719,17 +9939,49 @@ AST_MATCHER_P(Decl, hasDeclContext, internal::Matcher<Decl>, InnerMatcher) {
 ///   }
 ///   ns::A a;
 /// \endcode
-/// nestedNameSpecifier()
-///   matches "ns::" and both "A::"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{nestedNameSpecifier()}
+/// matches \match{type=name$ns} and both spellings of
+/// \match{type=name;count=2$A} in \c A::f() and \c ns::A .
 extern const internal::VariadicAllOfMatcher<NestedNameSpecifier>
     nestedNameSpecifier;
 
 /// Same as \c nestedNameSpecifier but matches \c NestedNameSpecifierLoc.
+///
+/// Given
+/// \code
+///   namespace ns {
+///     struct A { static void f(); };
+///     void A::f() {}
+///     void g() { A::f(); }
+///   }
+///   ns::A a;
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{nestedNameSpecifierLoc()} matches
+/// \match{count=2$A::} twice for the spellings in \c A::f() and \c ns::A ,
+/// and \match{ns::} once.
 extern const internal::VariadicAllOfMatcher<NestedNameSpecifierLoc>
     nestedNameSpecifierLoc;
 
 /// Matches \c NestedNameSpecifierLocs for which the given inner
 /// NestedNameSpecifier-matcher matches.
+///
+/// Given
+/// \code
+///   namespace ns {
+///     struct A { static void f(); };
+///     void A::f() {}
+///     void g() { A::f(); }
+///   }
+///   ns::A a;
+/// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{nestedNameSpecifierLoc(loc(specifiesType(
+/// hasDeclaration(namedDecl(hasName("A"))))))} matches \match{count=2$A::}
+/// twice for the spellings in \c A::f() and \c ns::A .
 AST_MATCHER_FUNCTION_P_OVERLOAD(
     internal::BindableMatcher<NestedNameSpecifierLoc>, loc,
     internal::Matcher<NestedNameSpecifier>, InnerMatcher, 1) {
@@ -7746,10 +9998,11 @@ AST_MATCHER_FUNCTION_P_OVERLOAD(
 ///   struct A { struct B { struct C {}; }; };
 ///   A::B::C c;
 /// \endcode
-/// nestedNameSpecifier(specifiesType(
+/// \compile_args{-std=c++}
+/// The matcher \matcher{nestedNameSpecifier(specifiesType(
 ///   hasDeclaration(cxxRecordDecl(hasName("A")))
-/// ))
-///   matches "A::"
+/// ))}
+/// matches the spelling of \match{type=name$A} in \c A::B::C .
 AST_MATCHER_P(NestedNameSpecifier, specifiesType,
               internal::Matcher<QualType>, InnerMatcher) {
   if (!Node.getAsType())
@@ -7765,9 +10018,10 @@ AST_MATCHER_P(NestedNameSpecifier, specifiesType,
 ///   struct A { struct B { struct C {}; }; };
 ///   A::B::C c;
 /// \endcode
-/// nestedNameSpecifierLoc(specifiesTypeLoc(loc(type(
-///   hasDeclaration(cxxRecordDecl(hasName("A")))))))
-///   matches "A::"
+/// \compile_args{-std=c++}
+/// The matcher \matcher{nestedNameSpecifierLoc(specifiesTypeLoc(loc(qualType(
+///   hasDeclaration(cxxRecordDecl(hasName("A")))))))}
+/// matches \match{A::}
 AST_MATCHER_P(NestedNameSpecifierLoc, specifiesTypeLoc,
               internal::Matcher<TypeLoc>, InnerMatcher) {
   return Node && Node.getNestedNameSpecifier()->getAsType() &&
@@ -7781,8 +10035,10 @@ AST_MATCHER_P(NestedNameSpecifierLoc, specifiesTypeLoc,
 ///   struct A { struct B { struct C {}; }; };
 ///   A::B::C c;
 /// \endcode
-/// nestedNameSpecifier(hasPrefix(specifiesType(asString("struct A")))) and
-///   matches "A::"
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{nestedNameSpecifier(hasPrefix(specifiesType(asString(
+/// "struct A"))))} matches \match{type=typestr$struct A::B}
 AST_MATCHER_P_OVERLOAD(NestedNameSpecifier, hasPrefix,
                        internal::Matcher<NestedNameSpecifier>, InnerMatcher,
                        0) {
@@ -7799,8 +10055,10 @@ AST_MATCHER_P_OVERLOAD(NestedNameSpecifier, hasPrefix,
 ///   struct A { struct B { struct C {}; }; };
 ///   A::B::C c;
 /// \endcode
-/// nestedNameSpecifierLoc(hasPrefix(loc(specifiesType(asString("struct A")))))
-///   matches "A::"
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{nestedNameSpecifierLoc(hasPrefix(loc(specifiesType(asString(
+/// "struct A")))))} matches \match{A::B::}.
 AST_MATCHER_P_OVERLOAD(NestedNameSpecifierLoc, hasPrefix,
                        internal::Matcher<NestedNameSpecifierLoc>, InnerMatcher,
                        1) {
@@ -7818,8 +10076,10 @@ AST_MATCHER_P_OVERLOAD(NestedNameSpecifierLoc, hasPrefix,
 ///   namespace ns { struct A {}; }
 ///   ns::A a;
 /// \endcode
-/// nestedNameSpecifier(specifiesNamespace(hasName("ns")))
-///   matches "ns::"
+/// \compile_args{-std=c++}
+/// The matcher
+/// \matcher{nestedNameSpecifier(specifiesNamespace(hasName("ns")))} matches
+/// the spelling of \match{type=name$ns} in \c ns::A .
 AST_MATCHER_P(NestedNameSpecifier, specifiesNamespace,
               internal::Matcher<NamespaceDecl>, InnerMatcher) {
   if (!Node.getAsNamespace())
@@ -7841,8 +10101,10 @@ AST_MATCHER_P(NestedNameSpecifier, specifiesNamespace,
 ///   #pragma omp declare simd
 ///   int min();
 /// \endcode
-/// attr()
-///   matches "nodiscard", "nonnull", "noinline", and the whole "#pragma" line.
+/// \compile_args{-fdeclspec;-fopenmp}
+/// The matcher \matcher{attr()}
+/// matches \match{nodiscard}, \match{nonnull}, \match{noinline}, and
+/// \match{declare simd}.
 extern const internal::VariadicAllOfMatcher<Attr> attr;
 
 /// Overloads for the \c equalsNode matcher.
@@ -7875,12 +10137,19 @@ AST_MATCHER_P_OVERLOAD(Type, equalsNode, const Type*, Other, 2) {
 ///
 /// Given
 /// \code
-///   switch (1) { case 1: case 2: default: switch (2) { case 3: case 4: ; } }
+///   void foo() {
+///     switch (1) { case 1: case 2: default: switch (2) { case 3: case 4: ; } }
+///   }
 /// \endcode
-/// switchStmt(forEachSwitchCase(caseStmt().bind("c"))).bind("s")
-///   matches four times, with "c" binding each of "case 1:", "case 2:",
-/// "case 3:" and "case 4:", and "s" respectively binding "switch (1)",
-/// "switch (1)", "switch (2)" and "switch (2)".
+/// The matcher
+/// \matcher{switchStmt(forEachSwitchCase(caseStmt().bind("c")))}
+/// matches four times, matching
+/// \match{count=2$switch (1) { case 1: case 2: default: switch (2) { case 3:
+/// case 4: ; } }} twice and
+/// \match{count=2$switch (2) { case 3: case 4: ; }} twice, with
+/// \matcher{type=sub$caseStmt()} matching each of \match{sub=c$case 1:},
+/// \match{sub=c$case 2:}, \match{sub=c$case 3:}
+/// and \match{sub=c$case 4:}.
 AST_MATCHER_P(SwitchStmt, forEachSwitchCase, internal::Matcher<SwitchCase>,
               InnerMatcher) {
   BoundNodesTreeBuilder Result;
@@ -7908,10 +10177,12 @@ AST_MATCHER_P(SwitchStmt, forEachSwitchCase, internal::Matcher<SwitchCase>,
 /// \code
 ///   class A { A() : i(42), j(42) {} int i; int j; };
 /// \endcode
-/// cxxConstructorDecl(forEachConstructorInitializer(
-///   forField(decl().bind("x"))
-/// ))
-///   will trigger two matches, binding for 'i' and 'j' respectively.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(forEachConstructorInitializer(
+///   forField(fieldDecl().bind("x"))))}
+/// matches the constructor of \match{count=2$A() : i(42), j(42) {}} twice, with
+/// \matcher{type=sub$fieldDecl()} matching \match{sub=field$i} and
+/// \match{sub=field$j} respectively.
 AST_MATCHER_P(CXXConstructorDecl, forEachConstructorInitializer,
               internal::Matcher<CXXCtorInitializer>, InnerMatcher) {
   BoundNodesTreeBuilder Result;
@@ -7939,7 +10210,10 @@ AST_MATCHER_P(CXXConstructorDecl, forEachConstructorInitializer,
 ///     S(S &&); // #3
 ///   };
 /// \endcode
-/// cxxConstructorDecl(isCopyConstructor()) will match #2, but not #1 or #3.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(isCopyConstructor())}
+/// matches \match{S(const S &)},
+/// but does not match \nomatch{S()} or \nomatch{S(S &&)}.
 AST_MATCHER(CXXConstructorDecl, isCopyConstructor) {
   return Node.isCopyConstructor();
 }
@@ -7954,7 +10228,10 @@ AST_MATCHER(CXXConstructorDecl, isCopyConstructor) {
 ///     S(S &&); // #3
 ///   };
 /// \endcode
-/// cxxConstructorDecl(isMoveConstructor()) will match #3, but not #1 or #2.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(isMoveConstructor())}
+/// matches \match{S(S &&)}
+/// but does not match \nomatch{S();} or \nomatch{S(S &&);}
 AST_MATCHER(CXXConstructorDecl, isMoveConstructor) {
   return Node.isMoveConstructor();
 }
@@ -7969,7 +10246,10 @@ AST_MATCHER(CXXConstructorDecl, isMoveConstructor) {
 ///     S(S &&); // #3
 ///   };
 /// \endcode
-/// cxxConstructorDecl(isDefaultConstructor()) will match #1, but not #2 or #3.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxConstructorDecl(isDefaultConstructor())}
+/// matches \match{S()}
+/// but does not match \nomatch{S(const S &);} or \nomatch{S(S &&);}.
 AST_MATCHER(CXXConstructorDecl, isDefaultConstructor) {
   return Node.isDefaultConstructor();
 }
@@ -7985,8 +10265,10 @@ AST_MATCHER(CXXConstructorDecl, isDefaultConstructor) {
 ///   };
 ///   S::S() : S(0) {} // #4
 /// \endcode
-/// cxxConstructorDecl(isDelegatingConstructor()) will match #3 and #4, but not
-/// #1 or #2.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{cxxConstructorDecl(isDelegatingConstructor())}
+/// matches \match{S(S &&) : S() {}} and \match{S::S() : S(0) {}},
+/// but does not match \nomatch{S()} or \nomatch{S(int)}.
 AST_MATCHER(CXXConstructorDecl, isDelegatingConstructor) {
   return Node.isDelegatingConstructor();
 }
@@ -8003,16 +10285,28 @@ AST_MATCHER(CXXConstructorDecl, isDelegatingConstructor) {
 ///     explicit S(double); // #2
 ///     operator int(); // #3
 ///     explicit operator bool(); // #4
-///     explicit(false) S(bool) // # 7
-///     explicit(true) S(char) // # 8
-///     explicit(b) S(S) // # 9
+///     explicit(false) S(bool); // # 7
+///     explicit(true) S(char); // # 8
+///     explicit(b) S(float); // # 9
 ///   };
-///   S(int) -> S<true> // #5
-///   explicit S(double) -> S<false> // #6
-/// \endcode
-/// cxxConstructorDecl(isExplicit()) will match #2 and #8, but not #1, #7 or #9.
-/// cxxConversionDecl(isExplicit()) will match #4, but not #3.
-/// cxxDeductionGuideDecl(isExplicit()) will match #6, but not #5.
+///   S(int) -> S<true>; // #5
+///   explicit S(double) -> S<false>; // #6
+/// \endcode
+/// \compile_args{-fno-delayed-template-parsing;-std=c++20-or-later}
+/// The matcher \matcher{cxxConstructorDecl(isExplicit())}
+/// matches \match{explicit S(double)}
+/// and \match{explicit(true) S(char)}
+/// but does not match \nomatch{S(int);}, \nomatch{explicit(false) S(bool);} or
+/// \nomatch{explicit(b) S(float)}
+/// The matcher \matcher{cxxConversionDecl(isExplicit())}
+/// matches \match{explicit operator bool()}
+/// but does not match \nomatch{operator int()}.
+/// The matcher \matcher{cxxDeductionGuideDecl(isExplicit())}
+/// matches the deduction guide \match{explicit S(double) -> S<false>},
+/// the implicit copy deduction candiate
+/// \match{type=typestr$auto (double) -> S<b>} and
+/// the implicitly generated deduction guide for \match{explicit(true) S(char)},
+/// but does not match \nomatch{S(int) -> S<true>}.
 AST_POLYMORPHIC_MATCHER(isExplicit, AST_POLYMORPHIC_SUPPORTED_TYPES(
                                         CXXConstructorDecl, CXXConversionDecl,
                                         CXXDeductionGuideDecl)) {
@@ -8030,16 +10324,28 @@ AST_POLYMORPHIC_MATCHER(isExplicit, AST_POLYMORPHIC_SUPPORTED_TYPES(
 ///     explicit S(double); // #2
 ///     operator int(); // #3
 ///     explicit operator bool(); // #4
-///     explicit(false) S(bool) // # 7
-///     explicit(true) S(char) // # 8
-///     explicit(b) S(S) // # 9
+///     explicit(false) S(bool); // # 7
+///     explicit(true) S(char); // # 8
+///     explicit(b) S(float); // # 9
 ///   };
-///   S(int) -> S<true> // #5
-///   explicit S(double) -> S<false> // #6
+///   S(int) -> S<true>; // #5
+///   explicit S(double) -> S<false>; // #6
 /// \endcode
-/// cxxConstructorDecl(hasExplicitSpecifier(constantExpr())) will match #7, #8 and #9, but not #1 or #2.
-/// cxxConversionDecl(hasExplicitSpecifier(constantExpr())) will not match #3 or #4.
-/// cxxDeductionGuideDecl(hasExplicitSpecifier(constantExpr())) will not match #5 or #6.
+/// \compile_args{-fno-delayed-template-parsing;-std=c++20-or-later}
+/// The matcher
+/// \matcher{cxxConstructorDecl(hasExplicitSpecifier(constantExpr()))} matches
+/// \match{explicit(false) S(bool)} and \match{explicit(true) S(char)},
+/// but does not match \nomatch{explicit(b) S(float)}, \nomatch{S(int)} or
+/// \nomatch{explicit S(double)}.
+/// The matcher
+/// \matcher{cxxConversionDecl(hasExplicitSpecifier(constantExpr()))} does not
+/// match \nomatch{operator int()} or \nomatch{explicit operator bool()}.
+/// Matcher
+/// The matcher
+/// \matcher{cxxDeductionGuideDecl(hasExplicitSpecifier(declRefExpr()))}
+/// matches the implicitly generated deduction guide
+/// \match{type=typestr$auto (float) -> S<b>} of the constructor
+/// \c{explicit(b) S(float)}.
 AST_MATCHER_P(FunctionDecl, hasExplicitSpecifier, internal::Matcher<Expr>,
               InnerMatcher) {
   ExplicitSpecifier ES = ExplicitSpecifier::getFromDecl(&Node);
@@ -8063,9 +10369,13 @@ AST_MATCHER_P(FunctionDecl, hasExplicitSpecifier, internal::Matcher<Expr>,
 ///   }
 ///   inline int Foo = 5;
 /// \endcode
-/// functionDecl(isInline()) will match ::f().
-/// namespaceDecl(isInline()) will match n::m.
-/// varDecl(isInline()) will match Foo;
+/// \compile_args{-std=c++}
+/// The matcher \matcher{functionDecl(isInline())} matches
+/// \match{inline void f()}.
+/// The matcher \matcher{namespaceDecl(isInline())} matches
+/// \match{inline namespace m {}}.
+/// The matcher \matcher{varDecl(isInline())} matches
+/// \match{inline int Foo = 5}.
 AST_POLYMORPHIC_MATCHER(isInline, AST_POLYMORPHIC_SUPPORTED_TYPES(NamespaceDecl,
                                                                   FunctionDecl,
                                                                   VarDecl)) {
@@ -8088,7 +10398,9 @@ AST_POLYMORPHIC_MATCHER(isInline, AST_POLYMORPHIC_SUPPORTED_TYPES(NamespaceDecl,
 ///   namespace {} // #1
 ///   }
 /// \endcode
-/// namespaceDecl(isAnonymous()) will match #1 but not ::n.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{namespaceDecl(isAnonymous())}
+/// matches \match{namespace {}}, but not \nomatch{namespace n}.
 AST_MATCHER(NamespaceDecl, isAnonymous) {
   return Node.isAnonymousNamespace();
 }
@@ -8113,7 +10425,9 @@ AST_MATCHER(NamespaceDecl, isAnonymous) {
 ///     }
 ///   }
 /// \endcode
-/// cxxRecordDecl(hasName("vector"), isInStdNamespace()) will match only #1.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("vector"), isInStdNamespace())}
+/// matches \match{class vector {}} inside of namespace std.
 AST_MATCHER(Decl, isInStdNamespace) { return Node.isInStdNamespace(); }
 
 /// Matches declarations in an anonymous namespace.
@@ -8130,12 +10444,16 @@ AST_MATCHER(Decl, isInStdNamespace) { return Node.isInStdNamespace(); }
 ///   namespace {
 ///     class vector {}; // #2
 ///     namespace foo {
-///       class vector{}; // #3
+///       class vector {}; // #3
 ///     }
 ///   }
 /// \endcode
-/// cxxRecordDecl(hasName("vector"), isInAnonymousNamespace()) will match
-/// #1, #2 and #3.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasName("vector"),
+///                         isInAnonymousNamespace())}
+/// matches \match{type=name;count=6$vector},
+/// two times for the definition and the implicit class declaration
+/// for each of the three definitions of \c vector .
 AST_MATCHER(Decl, isInAnonymousNamespace) {
   return Node.isInAnonymousNamespace();
 }
@@ -8145,10 +10463,13 @@ AST_MATCHER(Decl, isInAnonymousNamespace) {
 ///
 /// Given
 /// \code
-///   switch (1) { case 1: case 1+1: case 3 ... 4: ; }
+///   void foo() {
+///     switch (1) { case 1: break; case 1+1: break; case 3 ... 4: break; }
+///   }
 /// \endcode
-/// caseStmt(hasCaseConstant(integerLiteral()))
-///   matches "case 1:"
+/// The matcher
+/// \matcher{caseStmt(hasCaseConstant(constantExpr(has(integerLiteral()))))}
+/// matches \match{case 1: break}.
 AST_MATCHER_P(CaseStmt, hasCaseConstant, internal::Matcher<Expr>,
               InnerMatcher) {
   if (Node.getRHS())
@@ -8161,11 +10482,14 @@ AST_MATCHER_P(CaseStmt, hasCaseConstant, internal::Matcher<Expr>,
 ///
 /// Given
 /// \code
-///   __attribute__((device)) void f() { ... }
+///   __attribute__((device)) void f() {}
 /// \endcode
-/// decl(hasAttr(clang::attr::CUDADevice)) matches the function declaration of
-/// f. If the matcher is used from clang-query, attr::Kind parameter should be
-/// passed as a quoted string. e.g., hasAttr("attr::CUDADevice").
+/// \compile_args{--cuda-gpu-arch=sm_70;-std=c++}
+/// The matcher \matcher{decl(hasAttr(clang::attr::CUDADevice))}
+/// matches \match{__attribute__((device)) void f() {}}.
+/// If the matcher is used from clang-query, attr::Kind
+/// parameter should be passed as a quoted string. e.g.,
+/// \c hasAttr("attr::CUDADevice").
 AST_MATCHER_P(Decl, hasAttr, attr::Kind, AttrKind) {
   for (const auto *Attr : Node.attrs()) {
     if (Attr->getKind() == AttrKind)
@@ -8178,12 +10502,14 @@ AST_MATCHER_P(Decl, hasAttr, attr::Kind, AttrKind) {
 ///
 /// Given
 /// \code
-///   return a + b;
+///   int foo(int a, int b) {
+///     return a + b;
+///   }
 /// \endcode
-/// hasReturnValue(binaryOperator())
-///   matches 'return a + b'
-/// with binaryOperator()
-///   matching 'a + b'
+/// The matcher
+/// \matcher{returnStmt(hasReturnValue(binaryOperator().bind("op")))} matches
+/// \match{return a + b}, with \matcher{type=sub$binaryOperator()} matching
+/// \match{sub=op$a + b}.
 AST_MATCHER_P(ReturnStmt, hasReturnValue, internal::Matcher<Expr>,
               InnerMatcher) {
   if (const auto *RetValue = Node.getRetValue())
@@ -8193,18 +10519,25 @@ AST_MATCHER_P(ReturnStmt, hasReturnValue, internal::Matcher<Expr>,
 
 /// Matches CUDA kernel call expression.
 ///
-/// Example matches,
+/// Given
 /// \code
-///   kernel<<<i,j>>>();
+///   __global__ void kernel() {}
+///   void f() {
+///     kernel<<<32, 32>>>();
+///   }
 /// \endcode
+/// \compile_args{--cuda-gpu-arch=sm_70;-std=c++}
+/// The matcher \matcher{cudaKernelCallExpr()}
+/// matches \match{kernel<<<32, 32>>>()}
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, CUDAKernelCallExpr>
     cudaKernelCallExpr;
 
 /// Matches expressions that resolve to a null pointer constant, such as
 /// GNU's __null, C++11's nullptr, or C's NULL macro.
 ///
-/// Given:
+/// Given
 /// \code
+///   #define NULL 0
 ///   void *v1 = NULL;
 ///   void *v2 = nullptr;
 ///   void *v3 = __null; // GNU extension
@@ -8212,9 +10545,14 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, CUDAKernelCallExpr>
 ///   int *ip = 0;
 ///   int i = 0;
 /// \endcode
-/// expr(nullPointerConstant())
-///   matches the initializer for v1, v2, v3, cp, and ip. Does not match the
-///   initializer for i.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{expr(nullPointerConstant())}
+/// matches the initializer \match{NULL} of v1,
+/// matches the initializer \match{nullptr} of v2,
+/// matches the initializer \match{__null} of v3,
+/// matches the initializer \match{0} of cp and
+/// matches the initializer \match{0} of ip,
+/// but does not match the initializer \nomatch{i} of i.
 AST_MATCHER_FUNCTION(internal::Matcher<Expr>, nullPointerConstant) {
   return anyOf(
       gnuNullExpr(), cxxNullPtrLiteralExpr(),
@@ -8223,7 +10561,7 @@ AST_MATCHER_FUNCTION(internal::Matcher<Expr>, nullPointerConstant) {
 
 /// Matches the DecompositionDecl the binding belongs to.
 ///
-/// For example, in:
+/// Given
 /// \code
 /// void foo()
 /// {
@@ -8233,12 +10571,10 @@ AST_MATCHER_FUNCTION(internal::Matcher<Expr>, nullPointerConstant) {
 ///     f = 42;
 /// }
 /// \endcode
-/// The matcher:
-/// \code
-///   bindingDecl(hasName("f"),
-///                 forDecomposition(decompositionDecl())
-/// \endcode
-/// matches 'f' in 'auto &[f, s, t]'.
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{bindingDecl(hasName("f"),
+///                 forDecomposition(decompositionDecl()))}
+/// matches \match{type=name$f} in \c{auto &[f, s, t]}.
 AST_MATCHER_P(BindingDecl, forDecomposition, internal::Matcher<ValueDecl>,
               InnerMatcher) {
   if (const ValueDecl *VD = Node.getDecomposedDecl())
@@ -8248,7 +10584,7 @@ AST_MATCHER_P(BindingDecl, forDecomposition, internal::Matcher<ValueDecl>,
 
 /// Matches the Nth binding of a DecompositionDecl.
 ///
-/// For example, in:
+/// Given
 /// \code
 /// void foo()
 /// {
@@ -8258,12 +10594,10 @@ AST_MATCHER_P(BindingDecl, forDecomposition, internal::Matcher<ValueDecl>,
 ///     f = 42;
 /// }
 /// \endcode
-/// The matcher:
-/// \code
-///   decompositionDecl(hasBinding(0,
-///   bindingDecl(hasName("f").bind("fBinding"))))
-/// \endcode
-/// matches the decomposition decl with 'f' bound to "fBinding".
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{decompositionDecl(hasBinding(0,
+///   bindingDecl(hasName("f")).bind("fBinding")))}
+/// matches \match{auto &[f, s, t] = arr} with 'f' bound to "fBinding".
 AST_MATCHER_P2(DecompositionDecl, hasBinding, unsigned, N,
                internal::Matcher<BindingDecl>, InnerMatcher) {
   if (Node.bindings().size() <= N)
@@ -8283,11 +10617,10 @@ AST_MATCHER_P2(DecompositionDecl, hasBinding, unsigned, N,
 ///     f = 42;
 /// }
 /// \endcode
-/// The matcher:
-/// \code
-///   decompositionDecl(hasAnyBinding(bindingDecl(hasName("f").bind("fBinding"))))
-/// \endcode
-/// matches the decomposition decl with 'f' bound to "fBinding".
+/// \compile_args{-std=c++11-or-later}
+/// The matcher
+///  \matcher{decompositionDecl(hasAnyBinding(bindingDecl(hasName("f")).bind("fBinding")))}
+/// matches \match{auto &[f, s, t] = arr} with 'f' bound to "fBinding".
 AST_MATCHER_P(DecompositionDecl, hasAnyBinding, internal::Matcher<BindingDecl>,
               InnerMatcher) {
   return llvm::any_of(Node.bindings(), [&](const auto *Binding) {
@@ -8299,19 +10632,23 @@ AST_MATCHER_P(DecompositionDecl, hasAnyBinding, internal::Matcher<BindingDecl>,
 ///
 /// Deprecated. Use forCallable() to correctly handle the situation when
 /// the declaration is not a function (but a block or an Objective-C method).
-/// forFunction() not only fails to take non-functions into account but also
-/// may match the wrong declaration in their presence.
+/// The matcher \c forFunction() not only fails to take non-functions
+/// into account but also may match the wrong declaration in their presence.
 ///
-/// Given:
+/// Given
 /// \code
-/// F& operator=(const F& o) {
-///   std::copy_if(o.begin(), o.end(), begin(), [](V v) { return v > 0; });
-///   return *this;
-/// }
+///   struct F {
+///     F& operator=(const F& other) {
+///       []() { return 42 == 42; };
+///       return *this;
+///     }
+///   };
 /// \endcode
-/// returnStmt(forFunction(hasName("operator=")))
-///   matches 'return *this'
-///   but does not match 'return v > 0'
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{returnStmt(forFunction(hasName("operator=")))}
+/// matches \match{return *this}
+/// but does not match \nomatch{return 42 == 42}.
 AST_MATCHER_P(Stmt, forFunction, internal::Matcher<FunctionDecl>,
               InnerMatcher) {
   const auto &Parents = Finder->getASTContext().getParents(Node);
@@ -8339,30 +10676,34 @@ AST_MATCHER_P(Stmt, forFunction, internal::Matcher<FunctionDecl>,
 /// Matches declaration of the function, method, or block the statement
 /// belongs to.
 ///
-/// Given:
+/// Given
 /// \code
-/// F& operator=(const F& o) {
-///   std::copy_if(o.begin(), o.end(), begin(), [](V v) { return v > 0; });
-///   return *this;
-/// }
+/// struct F {
+///   F& operator=(const F& other) {
+///     []() { return 42 == 42; };
+///     return *this;
+///   }
+/// };
 /// \endcode
-/// returnStmt(forCallable(functionDecl(hasName("operator="))))
-///   matches 'return *this'
-///   but does not match 'return v > 0'
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{returnStmt(forFunction(hasName("operator=")))}
+/// matches \match{return *this}
+/// but does not match \nomatch{return 42 == 42}.
 ///
-/// Given:
+/// Given
 /// \code
-/// -(void) foo {
+/// void foo {
 ///   int x = 1;
 ///   dispatch_sync(queue, ^{ int y = 2; });
 /// }
 /// \endcode
-/// declStmt(forCallable(objcMethodDecl()))
-///   matches 'int x = 1'
-///   but does not match 'int y = 2'.
-/// whereas declStmt(forCallable(blockDecl()))
-///   matches 'int y = 2'
-///   but does not match 'int x = 1'.
+/// \compile_args{-ObjC}
+/// The matcher \matcher{declStmt(forCallable(objcMethodDecl()))}
+/// matches \match{int x = 1}
+/// but does not match \nomatch{int y = 2}.
+/// The matcher \matcher{declStmt(forCallable(blockDecl()))}
+/// matches \match{int y = 2}
+/// but does not match \nomatch{int x = 1}.
 AST_MATCHER_P(Stmt, forCallable, internal::Matcher<Decl>, InnerMatcher) {
   const auto &Parents = Finder->getASTContext().getParents(Node);
 
@@ -8404,73 +10745,93 @@ AST_MATCHER_P(Stmt, forCallable, internal::Matcher<Decl>, InnerMatcher) {
 
 /// Matches a declaration that has external formal linkage.
 ///
-/// Example matches only z (matcher = varDecl(hasExternalFormalLinkage()))
+/// Given
 /// \code
 /// void f() {
-///   int x;
-///   static int y;
+///   int a;
+///   static int b;
 /// }
-/// int z;
+/// int c;
+/// static int d;
 /// \endcode
+/// The matcher \matcher{varDecl(hasExternalFormalLinkage())}
+/// matches \match{int c},
+/// but not \nomatch{int a}, \nomatch{static int b} or \nomatch{int d}.
 ///
-/// Example matches f() because it has external formal linkage despite being
-/// unique to the translation unit as though it has internal likage
-/// (matcher = functionDecl(hasExternalFormalLinkage()))
-///
+/// Given
 /// \code
-/// namespace {
-/// void f() {}
-/// }
+///   namespace {
+///     void f() {}
+///   }
+///   void g() {}
+///   static void h() {}
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{functionDecl(hasExternalFormalLinkage())}
+/// matches \match{void g() {}}, but not \nomatch{void f() {}} or
+/// \nomatch{static void h() {}}.
 AST_MATCHER(NamedDecl, hasExternalFormalLinkage) {
   return Node.hasExternalFormalLinkage();
 }
 
 /// Matches a declaration that has default arguments.
 ///
-/// Example matches y (matcher = parmVarDecl(hasDefaultArgument()))
+/// Given
 /// \code
-/// void x(int val) {}
-/// void y(int val = 0) {}
+///   void x(int val) {}
+///   void y(int val = 0) {}
 /// \endcode
+/// \compile_args{-std=c++}
+///
+/// The matcher \matcher{parmVarDecl(hasDefaultArgument())}
+/// matches \match{int val = 0}.
 ///
 /// Deprecated. Use hasInitializer() instead to be able to
 /// match on the contents of the default argument.  For example:
 ///
+/// Given
 /// \code
-/// void x(int val = 7) {}
-/// void y(int val = 42) {}
+///   void x(int val = 7) {}
+///   void y(int val = 42) {}
 /// \endcode
-/// parmVarDecl(hasInitializer(integerLiteral(equals(42))))
-///   matches the parameter of y
+/// \compile_args{-std=c++}
 ///
-/// A matcher such as
-///   parmVarDecl(hasInitializer(anything()))
-/// is equivalent to parmVarDecl(hasDefaultArgument()).
+/// The matcher
+/// \matcher{parmVarDecl(hasInitializer(integerLiteral(equals(42))))},
+/// matches \match{int val = 42}.
 AST_MATCHER(ParmVarDecl, hasDefaultArgument) {
   return Node.hasDefaultArg();
 }
 
 /// Matches array new expressions.
 ///
-/// Given:
+/// Given
 /// \code
+///   struct MyClass { int x; };
 ///   MyClass *p1 = new MyClass[10];
 /// \endcode
-/// cxxNewExpr(isArray())
-///   matches the expression 'new MyClass[10]'.
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxNewExpr(isArray())}
+/// matches \match{new MyClass[10]}.
 AST_MATCHER(CXXNewExpr, isArray) {
   return Node.isArray();
 }
 
 /// Matches placement new expression arguments.
 ///
-/// Given:
+/// Given
 /// \code
-///   MyClass *p1 = new (Storage, 16) MyClass();
+///   void *operator new(decltype(sizeof(void*)), int, void*);
+///   struct MyClass { int x; };
+///   unsigned char Storage[sizeof(MyClass) * 10];
+///   MyClass *p1 = new (16, Storage) MyClass();
 /// \endcode
-/// cxxNewExpr(hasPlacementArg(1, integerLiteral(equals(16))))
-///   matches the expression 'new (Storage, 16) MyClass()'.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{cxxNewExpr(hasPlacementArg(0,
+///                       integerLiteral(equals(16))))}
+/// matches \match{new (16, Storage) MyClass()}.
 AST_MATCHER_P2(CXXNewExpr, hasPlacementArg, unsigned, Index,
                internal::Matcher<Expr>, InnerMatcher) {
   return Node.getNumPlacementArgs() > Index &&
@@ -8479,12 +10840,17 @@ AST_MATCHER_P2(CXXNewExpr, hasPlacementArg, unsigned, Index,
 
 /// Matches any placement new expression arguments.
 ///
-/// Given:
+/// Given
 /// \code
+///   void* operator new(decltype(sizeof(void*)), void*);
+///   struct MyClass { int x; };
+///   unsigned char Storage[sizeof(MyClass) * 10];
 ///   MyClass *p1 = new (Storage) MyClass();
 /// \endcode
-/// cxxNewExpr(hasAnyPlacementArg(anything()))
-///   matches the expression 'new (Storage, 16) MyClass()'.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher \matcher{cxxNewExpr(hasAnyPlacementArg(anything()))}
+/// matches \match{new (Storage) MyClass()}.
 AST_MATCHER_P(CXXNewExpr, hasAnyPlacementArg, internal::Matcher<Expr>,
               InnerMatcher) {
   return llvm::any_of(Node.placement_arguments(), [&](const Expr *Arg) {
@@ -8494,12 +10860,18 @@ AST_MATCHER_P(CXXNewExpr, hasAnyPlacementArg, internal::Matcher<Expr>,
 
 /// Matches array new expressions with a given array size.
 ///
-/// Given:
+/// Given
 /// \code
+///   void* operator new(decltype(sizeof(void*)));
+///   struct MyClass { int x; };
 ///   MyClass *p1 = new MyClass[10];
 /// \endcode
-/// cxxNewExpr(hasArraySize(integerLiteral(equals(10))))
-///   matches the expression 'new MyClass[10]'.
+/// \compile_args{-std=c++11-or-later}
+///
+/// The matcher
+/// \matcher{cxxNewExpr(hasArraySize(
+///             ignoringImplicit(integerLiteral(equals(10)))))}
+/// matches \match{new MyClass[10]}.
 AST_MATCHER_P(CXXNewExpr, hasArraySize, internal::Matcher<Expr>, InnerMatcher) {
   return Node.isArray() && *Node.getArraySize() &&
          InnerMatcher.matches(**Node.getArraySize(), Finder, Builder);
@@ -8507,33 +10879,42 @@ AST_MATCHER_P(CXXNewExpr, hasArraySize, internal::Matcher<Expr>, InnerMatcher) {
 
 /// Matches a class declaration that is defined.
 ///
-/// Example matches x (matcher = cxxRecordDecl(hasDefinition()))
+/// Given
 /// \code
 /// class x {};
 /// class y;
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{cxxRecordDecl(hasDefinition())}
+/// matches \match{class x {}}
 AST_MATCHER(CXXRecordDecl, hasDefinition) {
   return Node.hasDefinition();
 }
 
 /// Matches C++11 scoped enum declaration.
 ///
-/// Example matches Y (matcher = enumDecl(isScoped()))
+/// Given
 /// \code
 /// enum X {};
 /// enum class Y {};
 /// \endcode
+/// \compile_args{-std=c++}
+/// The matcher \matcher{enumDecl(isScoped())}
+/// matches \match{enum class Y {}}
 AST_MATCHER(EnumDecl, isScoped) {
   return Node.isScoped();
 }
 
 /// Matches a function declared with a trailing return type.
 ///
-/// Example matches Y (matcher = functionDecl(hasTrailingReturn()))
+/// Given
 /// \code
 /// int X() {}
 /// auto Y() -> int {}
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
+/// The matcher \matcher{functionDecl(hasTrailingReturn())}
+/// matches \match{auto Y() -> int {}}.
 AST_MATCHER(FunctionDecl, hasTrailingReturn) {
   if (const auto *F = Node.getType()->getAs<FunctionProtoType>())
     return F->hasTrailingReturn();
@@ -8552,7 +10933,6 @@ AST_MATCHER(FunctionDecl, hasTrailingReturn) {
 /// appear in the C++17 AST.
 ///
 /// Given
-///
 /// \code
 /// struct H {};
 /// H G();
@@ -8560,9 +10940,11 @@ AST_MATCHER(FunctionDecl, hasTrailingReturn) {
 ///   H D = G();
 /// }
 /// \endcode
+/// \compile_args{-std=c++11-or-later}
 ///
-/// ``varDecl(hasInitializer(ignoringElidableConstructorCall(callExpr())))``
-/// matches ``H D = G()`` in C++11 through C++17 (and beyond).
+/// The matcher
+/// \matcher{varDecl(hasInitializer(ignoringElidableConstructorCall(callExpr())))}
+/// matches \match{H D = G()}.
 AST_MATCHER_P(Expr, ignoringElidableConstructorCall, internal::Matcher<Expr>,
               InnerMatcher) {
   // E tracks the node that we are examining.
@@ -8593,15 +10975,21 @@ AST_MATCHER_P(Expr, ignoringElidableConstructorCall, internal::Matcher<Expr>,
 /// Matches any ``#pragma omp`` executable directive.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   #pragma omp parallel default(none)
-///   #pragma omp taskyield
+///   void foo() {
+///     #pragma omp parallel
+///       {}
+///     #pragma omp parallel default(none)
+///       {
+///         #pragma omp taskyield
+///       }
+///   }
 /// \endcode
-///
-/// ``ompExecutableDirective()`` matches ``omp parallel``,
-/// ``omp parallel default(none)`` and ``omp taskyield``.
+/// \compile_args{-fopenmp}
+/// The matcher \matcher{ompExecutableDirective()}
+/// matches \match{#pragma omp parallel},
+/// \match{#pragma omp parallel default(none)}
+/// and \match{#pragma omp taskyield}.
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPExecutableDirective>
     ompExecutableDirective;
 
@@ -8609,15 +10997,18 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPExecutableDirective>
 /// i.e., directives that can't have a structured block.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   {}
-///   #pragma omp taskyield
+///   void foo() {
+///     #pragma omp parallel
+///     {
+///       #pragma omp taskyield
+///     }
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompExecutableDirective(isStandaloneDirective()))`` matches
-/// ``omp taskyield``.
+/// The matcher \matcher{ompExecutableDirective(isStandaloneDirective())}
+/// matches \match{#pragma omp taskyield}.
 AST_MATCHER(OMPExecutableDirective, isStandaloneDirective) {
   return Node.isStandaloneDirective();
 }
@@ -8628,15 +11019,20 @@ AST_MATCHER(OMPExecutableDirective, isStandaloneDirective) {
 /// If it is, it will never match.
 ///
 /// Given
-///
 /// \code
-///    #pragma omp parallel
-///    ;
-///    #pragma omp parallel
-///    {}
+///   void foo() {
+///     #pragma omp parallel
+///     ;
+///     #pragma omp parallel
+///     {}
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompExecutableDirective(hasStructuredBlock(nullStmt()))`` will match ``;``
+/// The matcher
+/// \matcher{ompExecutableDirective(hasStructuredBlock(nullStmt().bind("stmt")))}
+/// matches \match{#pragma omp parallel},
+/// with \matcher{type=sub$stmtt()} matching \match{sub=stmt${}}.
 AST_MATCHER_P(OMPExecutableDirective, hasStructuredBlock,
               internal::Matcher<Stmt>, InnerMatcher) {
   if (Node.isStandaloneDirective())
@@ -8647,14 +11043,18 @@ AST_MATCHER_P(OMPExecutableDirective, hasStructuredBlock,
 /// Matches any clause in an OpenMP directive.
 ///
 /// Given
-///
 /// \code
+///   void foo() {
 ///   #pragma omp parallel
+///     ;
 ///   #pragma omp parallel default(none)
+///     ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompExecutableDirective(hasAnyClause(anything()))`` matches
-/// ``omp parallel default(none)``.
+/// The matcher \matcher{ompExecutableDirective(hasAnyClause(anything()))}
+/// matches \match{#pragma omp parallel default(none)}.
 AST_MATCHER_P(OMPExecutableDirective, hasAnyClause,
               internal::Matcher<OMPClause>, InnerMatcher) {
   ArrayRef<OMPClause *> Clauses = Node.clauses();
@@ -8666,33 +11066,53 @@ AST_MATCHER_P(OMPExecutableDirective, hasAnyClause,
 /// Matches OpenMP ``default`` clause.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel default(none)
-///   #pragma omp parallel default(shared)
-///   #pragma omp parallel default(private)
-///   #pragma omp parallel default(firstprivate)
-///   #pragma omp parallel
+///   void foo() {
+///     #pragma omp parallel default(none)
+///       ;
+///     #pragma omp parallel default(shared)
+///       ;
+///     #pragma omp parallel default(private)
+///       ;
+///     #pragma omp parallel default(firstprivate)
+///       ;
+///     #pragma omp parallel
+///       ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompDefaultClause()`` matches ``default(none)``, ``default(shared)``,
-/// `` default(private)`` and ``default(firstprivate)``
+/// The matcher
+/// \matcher{ompExecutableDirective(hasAnyClause(ompDefaultClause()))} matches
+/// \match{#pragma omp parallel default(none)},
+/// \match{#pragma omp parallel default(shared)},
+/// \match{#pragma omp parallel default(private)} and
+/// \match{#pragma omp parallel default(firstprivate)}.
 extern const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
     ompDefaultClause;
 
 /// Matches if the OpenMP ``default`` clause has ``none`` kind specified.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   #pragma omp parallel default(none)
-///   #pragma omp parallel default(shared)
-///   #pragma omp parallel default(private)
-///   #pragma omp parallel default(firstprivate)
+///   void foo() {
+///     #pragma omp parallel
+///       ;
+///     #pragma omp parallel default(none)
+///       ;
+///     #pragma omp parallel default(shared)
+///       ;
+///     #pragma omp parallel default(private)
+///       ;
+///     #pragma omp parallel default(firstprivate)
+///       ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompDefaultClause(isNoneKind())`` matches only ``default(none)``.
+/// The matcher
+/// \matcher{ompExecutableDirective(hasAnyClause(ompDefaultClause(isNoneKind())))}
+/// matches only \match{#pragma omp parallel default(none)}.
 AST_MATCHER(OMPDefaultClause, isNoneKind) {
   return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_none;
 }
@@ -8700,16 +11120,25 @@ AST_MATCHER(OMPDefaultClause, isNoneKind) {
 /// Matches if the OpenMP ``default`` clause has ``shared`` kind specified.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   #pragma omp parallel default(none)
+///   void foo() {
+///     #pragma omp parallel
+///       ;
+///     #pragma omp parallel default(none)
+///       ;
 ///   #pragma omp parallel default(shared)
+///       ;
 ///   #pragma omp parallel default(private)
+///       ;
 ///   #pragma omp parallel default(firstprivate)
+///       ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompDefaultClause(isSharedKind())`` matches only ``default(shared)``.
+/// The matcher
+/// \matcher{ompExecutableDirective(hasAnyClause(ompDefaultClause(isSharedKind())))}
+/// matches \match{#pragma omp parallel default(shared)}.
 AST_MATCHER(OMPDefaultClause, isSharedKind) {
   return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_shared;
 }
@@ -8718,17 +11147,25 @@ AST_MATCHER(OMPDefaultClause, isSharedKind) {
 /// specified.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
+///   void foo() {
+///     #pragma omp parallel
+///       ;
 ///   #pragma omp parallel default(none)
+///       ;
 ///   #pragma omp parallel default(shared)
+///       ;
 ///   #pragma omp parallel default(private)
+///       ;
 ///   #pragma omp parallel default(firstprivate)
+///       ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompDefaultClause(isPrivateKind())`` matches only
-/// ``default(private)``.
+/// The matcher
+/// \matcher{ompExecutableDirective(hasAnyClause(ompDefaultClause(isPrivateKind())))}
+/// matches \match{#pragma omp parallel default(private)}.
 AST_MATCHER(OMPDefaultClause, isPrivateKind) {
   return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_private;
 }
@@ -8737,17 +11174,25 @@ AST_MATCHER(OMPDefaultClause, isPrivateKind) {
 /// specified.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   #pragma omp parallel default(none)
-///   #pragma omp parallel default(shared)
-///   #pragma omp parallel default(private)
-///   #pragma omp parallel default(firstprivate)
+///   void foo() {
+///     #pragma omp parallel
+///       ;
+///     #pragma omp parallel default(none)
+///       ;
+///     #pragma omp parallel default(shared)
+///       ;
+///     #pragma omp parallel default(private)
+///       ;
+///     #pragma omp parallel default(firstprivate)
+///       ;
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// ``ompDefaultClause(isFirstPrivateKind())`` matches only
-/// ``default(firstprivate)``.
+/// The matcher
+/// \matcher{ompExecutableDirective(hasAnyClause(ompDefaultClause(isFirstPrivateKind())))}
+/// matches \match{#pragma omp parallel default(firstprivate)}.
 AST_MATCHER(OMPDefaultClause, isFirstPrivateKind) {
   return Node.getDefaultKind() == llvm::omp::OMP_DEFAULT_firstprivate;
 }
@@ -8756,15 +11201,23 @@ AST_MATCHER(OMPDefaultClause, isFirstPrivateKind) {
 /// clause kind.
 ///
 /// Given
-///
 /// \code
-///   #pragma omp parallel
-///   #pragma omp parallel for
-///   #pragma omp          for
+///   void foo() {
+///     #pragma omp parallel
+///       ;
+///     #pragma omp parallel for
+///       for (int i = 0; i < 10; ++i) {}
+///     #pragma omp          for
+///       for (int i = 0; i < 10; ++i) {}
+///   }
 /// \endcode
+/// \compile_args{-fopenmp}
 ///
-/// `ompExecutableDirective(isAllowedToContainClause(OMPC_default))`` matches
-/// ``omp parallel`` and ``omp parallel for``.
+/// The matcher
+/// \matcher{ompExecutableDirective(isAllowedToContainClauseKind(
+/// OpenMPClauseKind::OMPC_default))}
+/// matches \match{#pragma omp parallel}
+/// and \match{#pragma omp parallel for}.
 ///
 /// If the matcher is use from clang-query, ``OpenMPClauseKind`` parameter
 /// should be passed as a quoted string. e.g.,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index ce86116680d7a..fbcbf0ed41641 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3366,16 +3366,18 @@ def DiagnoseIf : InheritableAttr {
   let Spellings = [GNU<"diagnose_if">];
   let Subjects = SubjectList<[Function, ObjCMethod, ObjCProperty]>;
   let Args = [ExprArgument<"Cond">, StringArgument<"Message">,
-              EnumArgument<"DefaultSeverity",
-                           "DefaultSeverity",
+              EnumArgument<"DiagnosticType", "DiagnosticType",
                            /*is_string=*/true,
-                           ["error",    "warning"],
-                           ["DS_error", "DS_warning"]>,
-              StringArgument<"WarningGroup", /*optional*/ 1>,
+                           ["error", "warning"],
+                           ["DT_Error", "DT_Warning"]>,
               BoolArgument<"ArgDependent", 0, /*fake*/ 1>,
               DeclArgument<Named, "Parent", 0, /*fake*/ 1>];
   let InheritEvenIfAlreadyPresent = 1;
   let LateParsed = LateAttrParseStandard;
+  let AdditionalMembers = [{
+    bool isError() const { return diagnosticType == DT_Error; }
+    bool isWarning() const { return diagnosticType == DT_Warning; }
+  }];
   let TemplateDependent = 1;
   let Documentation = [DiagnoseIfDocs];
 }
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 8c5d7ad763bf9..33791270800c9 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -4703,12 +4703,6 @@ def HLSLClamp : LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void(...)";
 }
 
-def HLSLCreateHandle : LangBuiltin<"HLSL_LANG"> {
-  let Spellings = ["__builtin_hlsl_create_handle"];
-  let Attributes = [NoThrow, Const];
-  let Prototype = "void*(unsigned char)";
-}
-
 def HLSLDotProduct : LangBuiltin<"HLSL_LANG"> {
   let Spellings = ["__builtin_hlsl_dot"];
   let Attributes = [NoThrow, Const];
diff --git a/clang/include/clang/Basic/CFProtectionOptions.def b/clang/include/clang/Basic/CFProtectionOptions.def
new file mode 100644
index 0000000000000..b9df2de7f7eba
--- /dev/null
+++ b/clang/include/clang/Basic/CFProtectionOptions.def
@@ -0,0 +1,15 @@
+//===-- CFProtectionOptions.def - cf-protection options ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+
+#ifdef CF_BRANCH_LABEL_SCHEME
+CF_BRANCH_LABEL_SCHEME(Unlabeled, unlabeled)
+CF_BRANCH_LABEL_SCHEME(FuncSig, func-sig)
+
+#undef CF_BRANCH_LABEL_SCHEME
+#endif // #ifdef CF_BRANCH_LABEL_SCHEME
diff --git a/clang/include/clang/Basic/CFProtectionOptions.h b/clang/include/clang/Basic/CFProtectionOptions.h
new file mode 100644
index 0000000000000..13f46d4c13e7e
--- /dev/null
+++ b/clang/include/clang/Basic/CFProtectionOptions.h
@@ -0,0 +1,38 @@
+//===--- CFProtectionOptions.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines constants for -fcf-protection and other related flags.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_BASIC_CFPROTECTIONOPTIONS_H
+#define LLVM_CLANG_BASIC_CFPROTECTIONOPTIONS_H
+
+#include "llvm/Support/ErrorHandling.h"
+
+namespace clang {
+
+enum class CFBranchLabelSchemeKind {
+  Default,
+#define CF_BRANCH_LABEL_SCHEME(Kind, FlagVal) Kind,
+#include "clang/Basic/CFProtectionOptions.def"
+};
+
+static inline const char *
+getCFBranchLabelSchemeFlagVal(const CFBranchLabelSchemeKind Scheme) {
+#define CF_BRANCH_LABEL_SCHEME(Kind, FlagVal)                                  \
+  if (Scheme == CFBranchLabelSchemeKind::Kind)                                 \
+    return #FlagVal;
+#include "clang/Basic/CFProtectionOptions.def"
+
+  llvm::report_fatal_error("invalid scheme");
+}
+
+} // namespace clang
+
+#endif // #ifndef LLVM_CLANG_BASIC_CFPROTECTIONOPTIONS_H
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 2893377e5a38b..eac831278ee20 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -111,6 +111,8 @@ CODEGENOPT(CFProtectionReturn , 1, 0) ///< if -fcf-protection is
                                       ///< set to full or return.
 CODEGENOPT(CFProtectionBranch , 1, 0) ///< if -fcf-protection is
                                       ///< set to full or branch.
+ENUM_CODEGENOPT(CFBranchLabelScheme, CFBranchLabelSchemeKind, 2,
+    CFBranchLabelSchemeKind::Default) ///< if -mcf-branch-label-scheme is set.
 CODEGENOPT(FunctionReturnThunks, 1, 0) ///< -mfunction-return={keep|thunk-extern}
 CODEGENOPT(IndirectBranchCSPrefix, 1, 0) ///< if -mindirect-branch-cs-prefix
                                          ///< is set.
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index f2a707a8ba8d7..2dcf98b465661 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_BASIC_CODEGENOPTIONS_H
 #define LLVM_CLANG_BASIC_CODEGENOPTIONS_H
 
+#include "clang/Basic/CFProtectionOptions.h"
 #include "clang/Basic/PointerAuthOptions.h"
 #include "clang/Basic/Sanitizers.h"
 #include "clang/Basic/XRayInstr.h"
@@ -107,18 +108,13 @@ class CodeGenOptions : public CodeGenOptionsBase {
 
   // This field stores one of the allowed values for the option
   // -fbasic-block-sections=.  The allowed values with this option are:
-  // {"labels", "all", "list=<file>", "none"}.
+  // {"all", "list=<file>", "none"}.
   //
-  // "labels":      Only generate basic block symbols (labels) for all basic
-  //                blocks, do not generate unique sections for basic blocks.
-  //                Use the machine basic block id in the symbol name to
-  //                associate profile info from virtual address to machine
-  //                basic block.
   // "all" :        Generate basic block sections for all basic blocks.
   // "list=<file>": Generate basic block sections for a subset of basic blocks.
   //                The functions and the machine basic block ids are specified
   //                in the file.
-  // "none":        Disable sections/labels for basic blocks.
+  // "none":        Disable sections for basic blocks.
   std::string BBSections;
 
   // If set, override the default value of MCAsmInfo::BinutilsVersion. If
diff --git a/clang/include/clang/Basic/Diagnostic.h b/clang/include/clang/Basic/Diagnostic.h
index e17ed8f98afa9..3b1efdb12824c 100644
--- a/clang/include/clang/Basic/Diagnostic.h
+++ b/clang/include/clang/Basic/Diagnostic.h
@@ -371,12 +371,10 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
     // Map extensions to warnings or errors?
     diag::Severity ExtBehavior = diag::Severity::Ignored;
 
-    DiagnosticIDs &DiagIDs;
-
-    DiagState(DiagnosticIDs &DiagIDs)
+    DiagState()
         : IgnoreAllWarnings(false), EnableAllWarnings(false),
           WarningsAsErrors(false), ErrorsAsFatal(false),
-          SuppressSystemWarnings(false), DiagIDs(DiagIDs) {}
+          SuppressSystemWarnings(false) {}
 
     using iterator = llvm::DenseMap<unsigned, DiagnosticMapping>::iterator;
     using const_iterator =
@@ -886,8 +884,6 @@ class DiagnosticsEngine : public RefCountedBase<DiagnosticsEngine> {
   /// \param FormatString A fixed diagnostic format string that will be hashed
   /// and mapped to a unique DiagID.
   template <unsigned N>
-  // TODO: Deprecate this once all uses are removed from LLVM
-  // [[deprecated("Use a CustomDiagDesc instead of a Level")]]
   unsigned getCustomDiagID(Level L, const char (&FormatString)[N]) {
     return Diags->getCustomDiagID((DiagnosticIDs::Level)L,
                                   StringRef(FormatString, N - 1));
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index 21a307d1e8987..6a658cf14356f 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -96,6 +96,9 @@ def note_constexpr_pointer_constant_comparison : Note<
   "at runtime">;
 def note_constexpr_literal_comparison : Note<
   "comparison of addresses of literals has unspecified value">;
+def note_constexpr_opaque_call_comparison : Note<
+  "comparison against opaque constant address '%0' can only be performed at "
+  "runtime">;
 def note_constexpr_pointer_weak_comparison : Note<
   "comparison against address of weak declaration '%0' can only be performed "
   "at runtime">;
diff --git a/clang/include/clang/Basic/DiagnosticCategories.h b/clang/include/clang/Basic/DiagnosticCategories.h
index 839f8dee3ca89..14be326f7515f 100644
--- a/clang/include/clang/Basic/DiagnosticCategories.h
+++ b/clang/include/clang/Basic/DiagnosticCategories.h
@@ -21,12 +21,11 @@ namespace clang {
     };
 
     enum class Group {
-#define DIAG_ENTRY(GroupName, FlagNameOffset, Members, SubGroups, Docs)    \
-      GroupName,
+#define DIAG_ENTRY(GroupName, FlagNameOffset, Members, SubGroups, Docs)        \
+  GroupName,
 #include "clang/Basic/DiagnosticGroups.inc"
 #undef CATEGORY
 #undef DIAG_ENTRY
-      NUM_GROUPS
     };
   }  // end namespace diag
 }  // end namespace clang
diff --git a/clang/include/clang/Basic/DiagnosticIDs.h b/clang/include/clang/Basic/DiagnosticIDs.h
index 1fa38ed6066e2..a051af327de28 100644
--- a/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/clang/include/clang/Basic/DiagnosticIDs.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_CLANG_BASIC_DIAGNOSTICIDS_H
 #define LLVM_CLANG_BASIC_DIAGNOSTICIDS_H
 
-#include "clang/Basic/DiagnosticCategories.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
@@ -85,7 +84,7 @@ namespace clang {
     /// to either Ignore (nothing), Remark (emit a remark), Warning
     /// (emit a warning) or Error (emit as an error).  It allows clients to
     /// map ERRORs to Error or Fatal (stop emitting diagnostics after this one).
-    enum class Severity : uint8_t {
+    enum class Severity {
       // NOTE: 0 means "uncomputed".
       Ignored = 1, ///< Do not present this diagnostic, ignore it.
       Remark = 2,  ///< Present this diagnostic as a remark.
@@ -182,96 +181,13 @@ class DiagnosticMapping {
 class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
 public:
   /// The level of the diagnostic, after it has been through mapping.
-  enum Level : uint8_t { Ignored, Note, Remark, Warning, Error, Fatal };
-
-  // Diagnostic classes.
-  enum Class {
-    CLASS_INVALID = 0x00,
-    CLASS_NOTE = 0x01,
-    CLASS_REMARK = 0x02,
-    CLASS_WARNING = 0x03,
-    CLASS_EXTENSION = 0x04,
-    CLASS_ERROR = 0x05
-  };
-
-  static bool IsCustomDiag(diag::kind Diag) {
-    return Diag >= diag::DIAG_UPPER_LIMIT;
-  }
-
-  class CustomDiagDesc {
-    LLVM_PREFERRED_TYPE(diag::Severity)
-    unsigned DefaultSeverity : 3;
-    LLVM_PREFERRED_TYPE(Class)
-    unsigned DiagClass : 3;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ShowInSystemHeader : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ShowInSystemMacro : 1;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned HasGroup : 1;
-    diag::Group Group;
-    std::string Description;
-
-    auto get_as_tuple() const {
-      return std::tuple(DefaultSeverity, DiagClass, ShowInSystemHeader,
-                        ShowInSystemMacro, HasGroup, Group,
-                        std::string_view{Description});
-    }
-
-  public:
-    CustomDiagDesc(diag::Severity DefaultSeverity, std::string Description,
-                   unsigned Class = CLASS_WARNING,
-                   bool ShowInSystemHeader = false,
-                   bool ShowInSystemMacro = false,
-                   std::optional<diag::Group> Group = std::nullopt)
-        : DefaultSeverity(static_cast<unsigned>(DefaultSeverity)),
-          DiagClass(Class), ShowInSystemHeader(ShowInSystemHeader),
-          ShowInSystemMacro(ShowInSystemMacro), HasGroup(Group != std::nullopt),
-          Group(Group.value_or(diag::Group{})),
-          Description(std::move(Description)) {}
-
-    std::optional<diag::Group> GetGroup() const {
-      if (HasGroup)
-        return Group;
-      return std::nullopt;
-    }
-
-    diag::Severity GetDefaultSeverity() const {
-      return static_cast<diag::Severity>(DefaultSeverity);
-    }
-
-    Class GetClass() const { return static_cast<Class>(DiagClass); }
-    std::string_view GetDescription() const { return Description; }
-    bool ShouldShowInSystemHeader() const { return ShowInSystemHeader; }
-
-    friend bool operator==(const CustomDiagDesc &lhs,
-                           const CustomDiagDesc &rhs) {
-      return lhs.get_as_tuple() == rhs.get_as_tuple();
-    }
-
-    friend bool operator<(const CustomDiagDesc &lhs,
-                          const CustomDiagDesc &rhs) {
-      return lhs.get_as_tuple() < rhs.get_as_tuple();
-    }
-  };
-
-  struct GroupInfo {
-    LLVM_PREFERRED_TYPE(diag::Severity)
-    unsigned Severity : 3;
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned HasNoWarningAsError : 1;
+  enum Level {
+    Ignored, Note, Remark, Warning, Error, Fatal
   };
 
 private:
   /// Information for uniquing and looking up custom diags.
   std::unique_ptr<diag::CustomDiagInfo> CustomDiagInfo;
-  std::unique_ptr<GroupInfo[]> GroupInfos = []() {
-    auto GIs = std::make_unique<GroupInfo[]>(
-        static_cast<size_t>(diag::Group::NUM_GROUPS));
-    for (size_t i = 0; i != static_cast<size_t>(diag::Group::NUM_GROUPS); ++i)
-      GIs[i] = {{}, false};
-    return GIs;
-  }();
 
 public:
   DiagnosticIDs();
@@ -286,35 +202,7 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   // FIXME: Replace this function with a create-only facilty like
   // createCustomDiagIDFromFormatString() to enforce safe usage. At the time of
   // writing, nearly all callers of this function were invalid.
-  unsigned getCustomDiagID(CustomDiagDesc Diag);
-
-  // TODO: Deprecate this once all uses are removed from LLVM
-  // [[deprecated("Use a CustomDiagDesc instead of a Level")]]
-  unsigned getCustomDiagID(Level Level, StringRef Message) {
-    return getCustomDiagID([&]() -> CustomDiagDesc {
-      switch (Level) {
-      case DiagnosticIDs::Level::Ignored:
-        return {diag::Severity::Ignored, std::string(Message), CLASS_WARNING,
-                /*ShowInSystemHeader*/ true};
-      case DiagnosticIDs::Level::Note:
-        return {diag::Severity::Fatal, std::string(Message), CLASS_NOTE,
-                /*ShowInSystemHeader*/ true};
-      case DiagnosticIDs::Level::Remark:
-        return {diag::Severity::Remark, std::string(Message), CLASS_REMARK,
-                /*ShowInSystemHeader*/ true};
-      case DiagnosticIDs::Level::Warning:
-        return {diag::Severity::Warning, std::string(Message), CLASS_WARNING,
-                /*ShowInSystemHeader*/ true};
-      case DiagnosticIDs::Level::Error:
-        return {diag::Severity::Error, std::string(Message), CLASS_ERROR,
-                /*ShowInSystemHeader*/ true};
-      case DiagnosticIDs::Level::Fatal:
-        return {diag::Severity::Fatal, std::string(Message), CLASS_ERROR,
-                /*ShowInSystemHeader*/ true};
-      }
-      llvm_unreachable("Fully covered switch above!");
-    }());
-  }
+  unsigned getCustomDiagID(Level L, StringRef FormatString);
 
   //===--------------------------------------------------------------------===//
   // Diagnostic classification and reporting interfaces.
@@ -326,36 +214,35 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   /// Return true if the unmapped diagnostic levelof the specified
   /// diagnostic ID is a Warning or Extension.
   ///
-  /// This is not legal to call on NOTEs.
-  bool isWarningOrExtension(unsigned DiagID) const;
+  /// This only works on builtin diagnostics, not custom ones, and is not
+  /// legal to call on NOTEs.
+  static bool isBuiltinWarningOrExtension(unsigned DiagID);
 
   /// Return true if the specified diagnostic is mapped to errors by
   /// default.
-  bool isDefaultMappingAsError(unsigned DiagID) const;
+  static bool isDefaultMappingAsError(unsigned DiagID);
 
   /// Get the default mapping for this diagnostic.
-  DiagnosticMapping getDefaultMapping(unsigned DiagID) const;
-
-  void initCustomDiagMapping(DiagnosticMapping &, unsigned DiagID);
+  static DiagnosticMapping getDefaultMapping(unsigned DiagID);
 
-  /// Determine whether the given diagnostic ID is a Note.
-  bool isNote(unsigned DiagID) const;
+  /// Determine whether the given built-in diagnostic ID is a Note.
+  static bool isBuiltinNote(unsigned DiagID);
 
-  /// Determine whether the given diagnostic ID is for an
+  /// Determine whether the given built-in diagnostic ID is for an
   /// extension of some sort.
-  bool isExtensionDiag(unsigned DiagID) const {
+  static bool isBuiltinExtensionDiag(unsigned DiagID) {
     bool ignored;
-    return isExtensionDiag(DiagID, ignored);
+    return isBuiltinExtensionDiag(DiagID, ignored);
   }
 
-  /// Determine whether the given diagnostic ID is for an
+  /// Determine whether the given built-in diagnostic ID is for an
   /// extension of some sort, and whether it is enabled by default.
   ///
   /// This also returns EnabledByDefault, which is set to indicate whether the
   /// diagnostic is ignored by default (in which case -pedantic enables it) or
   /// treated as a warning/error by default.
   ///
-  bool isExtensionDiag(unsigned DiagID, bool &EnabledByDefault) const;
+  static bool isBuiltinExtensionDiag(unsigned DiagID, bool &EnabledByDefault);
 
   /// Given a group ID, returns the flag that toggles the group.
   /// For example, for Group::DeprecatedDeclarations, returns
@@ -365,22 +252,19 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   /// Given a diagnostic group ID, return its documentation.
   static StringRef getWarningOptionDocumentation(diag::Group GroupID);
 
-  void setGroupSeverity(StringRef Group, diag::Severity);
-  void setGroupNoWarningsAsError(StringRef Group, bool);
-
   /// Given a group ID, returns the flag that toggles the group.
   /// For example, for "deprecated-declarations", returns
   /// Group::DeprecatedDeclarations.
   static std::optional<diag::Group> getGroupForWarningOption(StringRef);
 
   /// Return the lowest-level group that contains the specified diagnostic.
-  std::optional<diag::Group> getGroupForDiag(unsigned DiagID) const;
+  static std::optional<diag::Group> getGroupForDiag(unsigned DiagID);
 
   /// Return the lowest-level warning option that enables the specified
   /// diagnostic.
   ///
   /// If there is no -Wfoo flag that controls the diagnostic, this returns null.
-  StringRef getWarningOptionForDiag(unsigned DiagID);
+  static StringRef getWarningOptionForDiag(unsigned DiagID);
 
   /// Return the category number that a specified \p DiagID belongs to,
   /// or 0 if no category.
@@ -481,8 +365,6 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
                         const DiagnosticsEngine &Diag) const LLVM_READONLY;
 
-  Class getDiagClass(unsigned DiagID) const;
-
   /// Used to report a diagnostic that is finally fully formed.
   ///
   /// \returns \c true if the diagnostic was emitted, \c false if it was
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index e4e04bff8b512..f3d5d4c56606c 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2933,15 +2933,9 @@ def ext_constexpr_function_never_constant_expr : ExtWarn<
   "constant expression">, InGroup<DiagGroup<"invalid-constexpr">>, DefaultError;
 def err_attr_cond_never_constant_expr : Error<
   "%0 attribute expression never produces a constant expression">;
-def err_diagnose_if_unknown_warning : Error<"unknown warning group '%0'">;
 def err_diagnose_if_invalid_diagnostic_type : Error<
   "invalid diagnostic type for 'diagnose_if'; use \"error\" or \"warning\" "
   "instead">;
-def err_diagnose_if_unknown_option : Error<"unknown diagnostic option">;
-def err_diagnose_if_expected_equals : Error<
-  "expected '=' after diagnostic option">;
-def err_diagnose_if_unexpected_value : Error<
-  "unexpected value; use 'true' or 'false'">;
 def err_constexpr_body_no_return : Error<
   "no return statement in %select{constexpr|consteval}0 function">;
 def err_constexpr_return_missing_expr : Error<
@@ -12394,6 +12388,7 @@ def err_hlsl_packoffset_alignment_mismatch : Error<"packoffset at 'y' not match
 def err_hlsl_pointers_unsupported : Error<
   "%select{pointers|references}0 are unsupported in HLSL">;
 def err_hlsl_missing_resource_class : Error<"HLSL resource needs to have [[hlsl::resource_class()]] attribute">;
+def err_hlsl_attribute_needs_intangible_type: Error<"attribute %0 can be used only on HLSL intangible type %1">;
 
 def err_hlsl_operator_unsupported : Error<
   "the '%select{&|*|->}0' operator is unsupported in HLSL">;
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index fd3346d29f26a..68db400c22e6c 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -364,6 +364,8 @@ BENIGN_LANGOPT(CompatibilityQualifiedIdBlockParamTypeChecking, 1, 0,
 LANGOPT(ObjCDisableDirectMethodsForTesting, 1, 0,
         "Disable recognition of objc_direct methods")
 LANGOPT(CFProtectionBranch , 1, 0, "Control-Flow Branch Protection enabled")
+ENUM_LANGOPT(CFBranchLabelScheme, CFBranchLabelSchemeKind, 2, CFBranchLabelSchemeKind::Default,
+             "Control-Flow Branch Protection Label Scheme")
 LANGOPT(FakeAddressSpaceMap , 1, 0, "OpenCL fake address space map")
 ENUM_LANGOPT(AddressSpaceMapMangling , AddrSpaceMapMangling, 2, ASMM_Target, "OpenCL address space map mangling mode")
 LANGOPT(IncludeDefaultHeader, 1, 0, "Include default header file for OpenCL")
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 6c186c410e158..c3d53ca92d450 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_BASIC_LANGOPTIONS_H
 #define LLVM_CLANG_BASIC_LANGOPTIONS_H
 
+#include "clang/Basic/CFProtectionOptions.h"
 #include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangStandard.h"
@@ -73,6 +74,7 @@ class LangOptionsBase {
 public:
   using Visibility = clang::Visibility;
   using RoundingMode = llvm::RoundingMode;
+  using CFBranchLabelSchemeKind = clang::CFBranchLabelSchemeKind;
 
   enum GCMode { NonGC, GCOnly, HybridGC };
   enum StackProtectorMode { SSPOff, SSPOn, SSPStrong, SSPReq };
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index f31d88a354ea2..5778385060629 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/CFProtectionOptions.h"
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
@@ -1727,6 +1728,13 @@ class TargetInfo : public TransferrableTargetInfo,
   virtual bool
   checkCFProtectionBranchSupported(DiagnosticsEngine &Diags) const;
 
+  /// Get the target default CFBranchLabelScheme scheme
+  virtual CFBranchLabelSchemeKind getDefaultCFBranchLabelScheme() const;
+
+  virtual bool
+  checkCFBranchLabelSchemeSupported(const CFBranchLabelSchemeKind Scheme,
+                                    DiagnosticsEngine &Diags) const;
+
   /// Check if the target supports CFProtection return.
   virtual bool
   checkCFProtectionReturnSupported(DiagnosticsEngine &Diags) const;
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index 92f39744f3d08..8652b5e3a9c90 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -2126,3 +2126,9 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "neon,faminmax" in {
   def FAMIN : WInst<"vamin", "...", "fhQdQfQh">;
   def FAMAX : WInst<"vamax", "...", "fhQdQfQh">;
 }
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in {
+  // fscale
+  def FSCALE_V128 : WInst<"vscale", "..(.S)", "QdQfQh">;
+  def FSCALE_V64 : WInst<"vscale", "(.q)(.q)(.qS)", "fh">;
+}
\ No newline at end of file
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 23bd686a85f52..8b8824c04a332 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1055,6 +1055,7 @@ def Xlinker : Separate<["-"], "Xlinker">, Flags<[LinkerInput, RenderAsInput]>,
   HelpText<"Pass <arg> to the linker">, MetaVarName<"<arg>">,
   Group<Link_Group>;
 def Xoffload_linker : JoinedAndSeparate<["-"], "Xoffload-linker">,
+  Visibility<[ClangOption, CLOption, FlangOption, DXCOption]>,
   HelpText<"Pass <arg> to the offload linkers or the ones identified by -<triple>">,
   MetaVarName<"<triple> <arg>">, Group<Link_Group>;
 def Xpreprocessor : Separate<["-"], "Xpreprocessor">, Group<Preprocessor_Group>,
@@ -1975,10 +1976,15 @@ def fno_color_diagnostics : Flag<["-"], "fno-color-diagnostics">, Group<f_Group>
   Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
   HelpText<"Disable colors in diagnostics">;
 def : Flag<["-"], "fdiagnostics-color">, Group<f_Group>,
-  Visibility<[ClangOption, CLOption, DXCOption]>, Alias<fcolor_diagnostics>;
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
+  Alias<fcolor_diagnostics>;
 def : Flag<["-"], "fno-diagnostics-color">, Group<f_Group>,
-  Visibility<[ClangOption, CLOption, DXCOption]>, Alias<fno_color_diagnostics>;
-def fdiagnostics_color_EQ : Joined<["-"], "fdiagnostics-color=">, Group<f_Group>;
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
+  Alias<fno_color_diagnostics>;
+def fdiagnostics_color_EQ : Joined<["-"], "fdiagnostics-color=">, Group<f_Group>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
+  Values<"auto,always,never">,
+  HelpText<"When to use colors in diagnostics">;
 def fansi_escape_codes : Flag<["-"], "fansi-escape-codes">, Group<f_Group>,
   Visibility<[ClangOption, CLOption, DXCOption, CC1Option]>,
   HelpText<"Use ANSI escape codes for diagnostics">,
@@ -2841,6 +2847,10 @@ def fcf_protection : Flag<["-"], "fcf-protection">, Group<f_Group>,
   Visibility<[ClangOption, CLOption, CC1Option]>,
   Alias<fcf_protection_EQ>, AliasArgs<["full"]>,
   HelpText<"Enable cf-protection in 'full' mode">;
+def mcf_branch_label_scheme_EQ : Joined<["-"], "mcf-branch-label-scheme=">,
+  Visibility<[ClangOption, CC1Option]>, Group<m_Group>,
+  HelpText<"Select label scheme for branch control-flow architecture protection">,
+  Values<"unlabeled,func-sig">;
 def mfunction_return_EQ : Joined<["-"], "mfunction-return=">,
   Group<m_Group>, Visibility<[ClangOption, CLOption, CC1Option]>,
   HelpText<"Replace returns with jumps to ``__x86_return_thunk`` (x86 only, error otherwise)">,
@@ -4244,8 +4254,8 @@ defm basic_block_address_map : BoolFOption<"basic-block-address-map",
 def fbasic_block_sections_EQ : Joined<["-"], "fbasic-block-sections=">, Group<f_Group>,
   Visibility<[ClangOption, CC1Option, CC1AsOption]>,
   HelpText<"Place each function's basic blocks in unique sections (ELF Only)">,
-  DocBrief<[{Generate labels for each basic block or place each basic block or a subset of basic blocks in its own section.}]>,
-  Values<"all,labels,none,list=">,
+  DocBrief<[{Place each basic block or a subset of basic blocks in its own section.}]>,
+  Values<"all,none,list=">,
   MarshallingInfoString<CodeGenOpts<"BBSections">, [{"none"}]>;
 defm data_sections : BoolFOption<"data-sections",
   CodeGenOpts<"DataSections">, DefaultFalse,
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index e088254c566d3..311cd58bbcac2 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -70,7 +70,7 @@ class SemaHLSL : public SemaBase {
   void handleShaderAttr(Decl *D, const ParsedAttr &AL);
   void handleResourceBindingAttr(Decl *D, const ParsedAttr &AL);
   void handleParamModifierAttr(Decl *D, const ParsedAttr &AL);
-  bool handleResourceTypeAttr(const ParsedAttr &AL);
+  bool handleResourceTypeAttr(QualType T, const ParsedAttr &AL);
 
   bool CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
   QualType ProcessResourceTypeAttributes(QualType Wrapped);
diff --git a/clang/include/clang/Sema/Template.h b/clang/include/clang/Sema/Template.h
index 0340c23fd170d..fe27290efdbfc 100644
--- a/clang/include/clang/Sema/Template.h
+++ b/clang/include/clang/Sema/Template.h
@@ -411,6 +411,11 @@ enum class TemplateSubstitutionKind : char {
     /// lookup will search our outer scope.
     bool CombineWithOuterScope;
 
+    /// Whether this scope is being used to instantiate a lambda expression,
+    /// in which case it should be reused for instantiating the lambda's
+    /// FunctionProtoType.
+    bool InstantiatingLambda = false;
+
     /// If non-NULL, the template parameter pack that has been
     /// partially substituted per C++0x [temp.arg.explicit]p9.
     NamedDecl *PartiallySubstitutedPack = nullptr;
@@ -425,9 +430,11 @@ enum class TemplateSubstitutionKind : char {
     unsigned NumArgsInPartiallySubstitutedPack;
 
   public:
-    LocalInstantiationScope(Sema &SemaRef, bool CombineWithOuterScope = false)
+    LocalInstantiationScope(Sema &SemaRef, bool CombineWithOuterScope = false,
+                            bool InstantiatingLambda = false)
         : SemaRef(SemaRef), Outer(SemaRef.CurrentInstantiationScope),
-          CombineWithOuterScope(CombineWithOuterScope) {
+          CombineWithOuterScope(CombineWithOuterScope),
+          InstantiatingLambda(InstantiatingLambda) {
       SemaRef.CurrentInstantiationScope = this;
     }
 
@@ -553,6 +560,9 @@ enum class TemplateSubstitutionKind : char {
 
     /// Determine whether D is a pack expansion created in this scope.
     bool isLocalPackExpansion(const Decl *D);
+
+    /// Determine whether this scope is for instantiating a lambda.
+    bool isLambda() const { return InstantiatingLambda; }
   };
 
   class TemplateDeclInstantiator
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 760866fd9de93..e21d41c867314 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -233,13 +233,13 @@ class ASTWriter : public ASTDeserializationListener,
   /// instead of comparing the result of `getDeclID()` or `GetDeclRef()`.
   llvm::SmallPtrSet<const Decl *, 32> PredefinedDecls;
 
-  /// Mapping from FunctionDecl to the list of lambda IDs inside the function.
+  /// Mapping from FunctionDecl ID to the list of lambda IDs inside the
+  /// function.
   ///
   /// These lambdas have to be loaded right after the function they belong to.
   /// In order to have canonical declaration for lambda class from the same
   /// module as enclosing function during deserialization.
-  llvm::DenseMap<const Decl *, SmallVector<LocalDeclID, 4>>
-      FunctionToLambdasMap;
+  llvm::DenseMap<LocalDeclID, SmallVector<LocalDeclID, 4>> FunctionToLambdasMap;
 
   /// Offset of each declaration in the bitstream, indexed by
   /// the declaration's ID.
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 3e920c0f68360..30e7f6b3e57bd 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -88,13 +88,13 @@ class InputFile {
 
   InputFile(FileEntryRef File, bool isOverridden = false,
             bool isOutOfDate = false) {
-    assert(!(isOverridden && isOutOfDate) &&
-           "an overridden cannot be out-of-date");
     unsigned intVal = 0;
-    if (isOverridden)
-      intVal = Overridden;
-    else if (isOutOfDate)
+    // Make isOutOfDate with higher priority than isOverridden.
+    // It is possible if the recorded hash value mismatches.
+    if (isOutOfDate)
       intVal = OutOfDate;
+    else if (isOverridden)
+      intVal = Overridden;
     Val.setPointerAndInt(&File.getMapEntry(), intVal);
   }
 
diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 585246547b3dc..747ebd8c2e4de 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -74,7 +74,6 @@ def Performance : Package<"performance">, ParentPackage<OptIn>;
 def Security : Package <"security">;
 def InsecureAPI : Package<"insecureAPI">, ParentPackage<Security>;
 def SecurityAlpha : Package<"security">, ParentPackage<Alpha>;
-def Taint : Package<"taint">, ParentPackage<SecurityAlpha>;
 
 def CERT : Package<"cert">, ParentPackage<Security>;
 def ENV : Package<"env">, ParentPackage<CERT>;
@@ -1049,30 +1048,6 @@ def ReturnPointerRangeChecker : Checker<"ReturnPtrRange">,
 
 } // end "alpha.security"
 
-//===----------------------------------------------------------------------===//
-// Taint checkers.
-//===----------------------------------------------------------------------===//
-
-let ParentPackage = Taint in {
-
-def TaintPropagationChecker : Checker<"TaintPropagation">, // Modelling checker
-  HelpText<"Generate taint information used by other checkers">,
-  CheckerOptions<[
-    CmdLineOption<String,
-                  "Config",
-                  "Specifies the name of the configuration file.",
-                  "",
-                  InAlpha>,
-  ]>,
-  Documentation<NotDocumented>,
-  Hidden;
-
-def GenericTaintChecker : Checker<"GenericTaint">,
-  HelpText<"Reports potential injection vulnerabilities">,
-  Dependencies<[TaintPropagationChecker]>,
-  Documentation<HasDocumentation>;
-
-} // end "alpha.security.taint"
 
 //===----------------------------------------------------------------------===//
 // Mac OS X, Cocoa, and Core Foundation checkers.
@@ -1704,6 +1679,24 @@ def UnixAPIPortabilityChecker : Checker<"UnixAPI">,
 
 let ParentPackage = TaintOptIn in {
 
+def TaintPropagationChecker : Checker<"TaintPropagation">, // Modelling checker
+  HelpText<"Generate taint information used by other checkers">,
+  CheckerOptions<[
+    CmdLineOption<String,
+                  "Config",
+                  "Specifies the name of the configuration file.",
+                  "",
+                  Released>
+  ]>,
+  Documentation<NotDocumented>,
+  Hidden;
+
+def GenericTaintChecker : Checker<"GenericTaint">,
+  HelpText<"Reports potential injection vulnerabilities">,
+  Dependencies<[TaintPropagationChecker]>,
+  Documentation<HasDocumentation>;
+
+
 def TaintedAllocChecker: Checker<"TaintedAlloc">,
   HelpText<"Check for memory allocations, where the size parameter "
            "might be a tainted (attacker controlled) value.">,
@@ -1771,6 +1764,10 @@ def UncountedLambdaCapturesChecker : Checker<"UncountedLambdaCapturesChecker">,
 
 let ParentPackage = WebKitAlpha in {
 
+def NoUncheckedPtrMemberChecker : Checker<"NoUncheckedPtrMemberChecker">,
+  HelpText<"Check for no unchecked member variables.">,
+  Documentation<HasDocumentation>;
+
 def UncountedCallArgsChecker : Checker<"UncountedCallArgsChecker">,
   HelpText<"Check uncounted call arguments.">,
   Documentation<HasDocumentation>;
diff --git a/clang/include/clang/Testing/CommandLineArgs.h b/clang/include/clang/Testing/CommandLineArgs.h
index e71907e8bbd0c..52beac7254fef 100644
--- a/clang/include/clang/Testing/CommandLineArgs.h
+++ b/clang/include/clang/Testing/CommandLineArgs.h
@@ -21,19 +21,18 @@
 namespace clang {
 
 enum TestLanguage {
-  Lang_C89,
-  Lang_C99,
-  Lang_CXX03,
-  Lang_CXX11,
-  Lang_CXX14,
-  Lang_CXX17,
-  Lang_CXX20,
-  Lang_CXX23,
+#define TESTLANGUAGE(lang, version, std_flag, version_index)                   \
+  Lang_##lang##version,
+#include "clang/Testing/TestLanguage.def"
+
   Lang_OpenCL,
   Lang_OBJC,
-  Lang_OBJCXX
+  Lang_OBJCXX,
 };
 
+std::vector<TestLanguage> getCOrLater(int MinimumStd);
+std::vector<TestLanguage> getCXXOrLater(int MinimumStd);
+
 std::vector<std::string> getCommandLineArgsForTesting(TestLanguage Lang);
 std::vector<std::string> getCC1ArgsForTesting(TestLanguage Lang);
 
diff --git a/clang/include/clang/Testing/TestClangConfig.h b/clang/include/clang/Testing/TestClangConfig.h
index 1b4efca80e9d4..e52aa37482dc1 100644
--- a/clang/include/clang/Testing/TestClangConfig.h
+++ b/clang/include/clang/Testing/TestClangConfig.h
@@ -27,37 +27,90 @@ struct TestClangConfig {
   /// The argument of the `-target` command line flag.
   std::string Target;
 
-  bool isC() const { return Language == Lang_C89 || Language == Lang_C99; }
+  bool isC() const {
+    return false
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  || Language == Lang_##lang##version
+#include "clang/Testing/TestLanguage.def"
+        ;
+  }
 
-  bool isC99OrLater() const { return Language == Lang_C99; }
+  bool isC(int Version) const {
+    return false
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  || (Version == version && Language == Lang_##lang##version)
+#include "clang/Testing/TestLanguage.def"
+        ;
+  }
 
-  bool isCXX() const {
-    return Language == Lang_CXX03 || Language == Lang_CXX11 ||
-           Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20 || Language == Lang_CXX23;
+  bool isCOrLater(int MinimumStdVersion) const {
+    const auto MinimumStdVersionIndex = 0
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  +(MinimumStdVersion == version ? version_index : 0)
+#include "clang/Testing/TestLanguage.def"
+        ;
+    switch (Language) {
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  case Lang_##lang##version:                                                   \
+    return MinimumStdVersionIndex <= version_index;
+#include "clang/Testing/TestLanguage.def"
+    default:
+      return false;
+    }
   }
 
-  bool isCXX11OrLater() const {
-    return Language == Lang_CXX11 || Language == Lang_CXX14 ||
-           Language == Lang_CXX17 || Language == Lang_CXX20 ||
-           Language == Lang_CXX23;
+  bool isC99OrLater() const { return isCOrLater(99); }
+
+  bool isCOrEarlier(int MaximumStdVersion) const {
+    return isC() && (isC(MaximumStdVersion) || !isCOrLater(MaximumStdVersion));
   }
 
-  bool isCXX14OrLater() const {
-    return Language == Lang_CXX14 || Language == Lang_CXX17 ||
-           Language == Lang_CXX20 || Language == Lang_CXX23;
+  bool isCXX() const {
+    return false
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  || Language == Lang_##lang##version
+#include "clang/Testing/TestLanguage.def"
+        ;
   }
 
-  bool isCXX17OrLater() const {
-    return Language == Lang_CXX17 || Language == Lang_CXX20 ||
-           Language == Lang_CXX23;
+  bool isCXX(int Version) const {
+    return false
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  || (Version == version && Language == Lang_##lang##version)
+#include "clang/Testing/TestLanguage.def"
+        ;
   }
 
-  bool isCXX20OrLater() const {
-    return Language == Lang_CXX20 || Language == Lang_CXX23;
+  bool isCXXOrLater(int MinimumStdVersion) const {
+    const auto MinimumStdVersionIndex = 0
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  +(MinimumStdVersion == version ? version_index : 0)
+#include "clang/Testing/TestLanguage.def"
+        ;
+    switch (Language) {
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  case Lang_##lang##version:                                                   \
+    return MinimumStdVersionIndex <= version_index;
+#include "clang/Testing/TestLanguage.def"
+    default:
+      return false;
+    }
   }
 
-  bool isCXX23OrLater() const { return Language == Lang_CXX23; }
+  bool isCXX11OrLater() const { return isCXXOrLater(11); }
+
+  bool isCXX14OrLater() const { return isCXXOrLater(14); }
+
+  bool isCXX17OrLater() const { return isCXXOrLater(17); }
+
+  bool isCXX20OrLater() const { return isCXXOrLater(20); }
+
+  bool isCXX23OrLater() const { return isCXXOrLater(23); }
+
+  bool isCXXOrEarlier(int MaximumStdVersion) const {
+    return isCXX() &&
+           (isCXX(MaximumStdVersion) || !isCXXOrLater(MaximumStdVersion));
+  }
 
   bool supportsCXXDynamicExceptionSpecification() const {
     return Language == Lang_CXX03 || Language == Lang_CXX11 ||
@@ -75,6 +128,30 @@ struct TestClangConfig {
     return Result;
   }
 
+  std::string toShortString() const {
+    std::string Result;
+    llvm::raw_string_ostream OS(Result);
+    switch (Language) {
+#define TESTLANGUAGE(lang, version, std_flag, version_index)                   \
+  case Lang_##lang##version:                                                   \
+    OS << (#lang #version);                                                    \
+    break;
+#include "clang/Testing/TestLanguage.def"
+    case Lang_OpenCL:
+      OS << "OpenCL";
+      break;
+    case Lang_OBJC:
+      OS << "OBJC";
+      break;
+    case Lang_OBJCXX:
+      OS << "OBJCXX";
+      break;
+    }
+
+    OS << (Target.find("win") != std::string::npos ? "_win" : "");
+    return Result;
+  }
+
   std::string toString() const {
     std::string Result;
     llvm::raw_string_ostream OS(Result);
diff --git a/clang/include/clang/Testing/TestLanguage.def b/clang/include/clang/Testing/TestLanguage.def
new file mode 100644
index 0000000000000..ac62b176a0b87
--- /dev/null
+++ b/clang/include/clang/Testing/TestLanguage.def
@@ -0,0 +1,47 @@
+
+//===-- TestLanguage.def - Language Versions for Testing --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// The TESTLANGUAGE(-C/-CXX) macros have four parameters:
+// the language, the standard version, the corresponding compile-flag,
+// and an index of the language version for each language.
+// The standard version is used to compare a standard version numerically,
+// and the index is used to impose ordering for the language versions
+// with respect to each language.
+
+#ifndef TESTLANGUAGE
+#  define TESTLANGUAGE(...)
+#endif
+
+#ifndef TESTLANGUAGE_C
+#  define TESTLANGUAGE_C(...) TESTLANGUAGE(__VA_ARGS__)
+#endif
+
+#ifndef TESTLANGUAGE_CXX
+#  define TESTLANGUAGE_CXX(...) TESTLANGUAGE(__VA_ARGS__)
+#endif
+
+TESTLANGUAGE_C(C, 89, c89, 0)
+TESTLANGUAGE_C(C, 99, c99, 1)
+TESTLANGUAGE_C(C, 11, c11, 2)
+TESTLANGUAGE_C(C, 17, c17, 3)
+TESTLANGUAGE_C(C, 23, c23, 4)
+TESTLANGUAGE_C(C, 26, c2y, 5)
+
+// TESTLANGUAGE_CXX(CXX, 98, c++98, 0)
+TESTLANGUAGE_CXX(CXX, 03, c++03, 1)
+TESTLANGUAGE_CXX(CXX, 11, c++11, 2)
+TESTLANGUAGE_CXX(CXX, 14, c++14, 3)
+TESTLANGUAGE_CXX(CXX, 17, c++17, 4)
+TESTLANGUAGE_CXX(CXX, 20, c++20, 5)
+TESTLANGUAGE_CXX(CXX, 23, c++23, 6)
+TESTLANGUAGE_CXX(CXX, 26, c++26, 7)
+
+#undef TESTLANGUAGE_CXX
+#undef TESTLANGUAGE_C
+#undef TESTLANGUAGE
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index fd8aa8de79b49..cda8b02cc8499 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -3535,6 +3535,50 @@ QualType ASTContext::getCountAttributedType(
   return QualType(CATy, 0);
 }
 
+QualType
+ASTContext::adjustType(QualType Orig,
+                       llvm::function_ref<QualType(QualType)> Adjust) const {
+  switch (Orig->getTypeClass()) {
+  case Type::Attributed: {
+    const auto *AT = dyn_cast<AttributedType>(Orig);
+    return getAttributedType(AT->getAttrKind(),
+                             adjustType(AT->getModifiedType(), Adjust),
+                             adjustType(AT->getEquivalentType(), Adjust));
+  }
+
+  case Type::BTFTagAttributed: {
+    const auto *BTFT = dyn_cast<BTFTagAttributedType>(Orig);
+    return getBTFTagAttributedType(BTFT->getAttr(),
+                                   adjustType(BTFT->getWrappedType(), Adjust));
+  }
+
+  case Type::Elaborated: {
+    const auto *ET = cast<ElaboratedType>(Orig);
+    return getElaboratedType(ET->getKeyword(), ET->getQualifier(),
+                             adjustType(ET->getNamedType(), Adjust));
+  }
+
+  case Type::Paren:
+    return getParenType(
+        adjustType(cast<ParenType>(Orig)->getInnerType(), Adjust));
+
+  case Type::Adjusted: {
+    const auto *AT = cast<AdjustedType>(Orig);
+    return getAdjustedType(AT->getOriginalType(),
+                           adjustType(AT->getAdjustedType(), Adjust));
+  }
+
+  case Type::MacroQualified: {
+    const auto *MQT = cast<MacroQualifiedType>(Orig);
+    return getMacroQualifiedType(adjustType(MQT->getUnderlyingType(), Adjust),
+                                 MQT->getMacroIdentifier());
+  }
+
+  default:
+    return Adjust(Orig);
+  }
+}
+
 const FunctionType *ASTContext::adjustFunctionType(const FunctionType *T,
                                                    FunctionType::ExtInfo Info) {
   if (T->getExtInfo() == Info)
@@ -3553,13 +3597,23 @@ const FunctionType *ASTContext::adjustFunctionType(const FunctionType *T,
   return cast<FunctionType>(Result.getTypePtr());
 }
 
+QualType ASTContext::adjustFunctionResultType(QualType FunctionType,
+                                              QualType ResultType) {
+  return adjustType(FunctionType, [&](QualType Orig) {
+    if (const auto *FNPT = Orig->getAs<FunctionNoProtoType>())
+      return getFunctionNoProtoType(ResultType, FNPT->getExtInfo());
+
+    const auto *FPT = Orig->castAs<FunctionProtoType>();
+    return getFunctionType(ResultType, FPT->getParamTypes(),
+                           FPT->getExtProtoInfo());
+  });
+}
+
 void ASTContext::adjustDeducedFunctionResultType(FunctionDecl *FD,
                                                  QualType ResultType) {
   FD = FD->getMostRecentDecl();
   while (true) {
-    const auto *FPT = FD->getType()->castAs<FunctionProtoType>();
-    FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
-    FD->setType(getFunctionType(ResultType, FPT->getParamTypes(), EPI));
+    FD->setType(adjustFunctionResultType(FD->getType(), ResultType));
     if (FunctionDecl *Next = FD->getPreviousDecl())
       FD = Next;
     else
@@ -3575,30 +3629,11 @@ void ASTContext::adjustDeducedFunctionResultType(FunctionDecl *FD,
 /// and preserved. Other type sugar (for instance, typedefs) is not.
 QualType ASTContext::getFunctionTypeWithExceptionSpec(
     QualType Orig, const FunctionProtoType::ExceptionSpecInfo &ESI) const {
-  // Might have some parens.
-  if (const auto *PT = dyn_cast<ParenType>(Orig))
-    return getParenType(
-        getFunctionTypeWithExceptionSpec(PT->getInnerType(), ESI));
-
-  // Might be wrapped in a macro qualified type.
-  if (const auto *MQT = dyn_cast<MacroQualifiedType>(Orig))
-    return getMacroQualifiedType(
-        getFunctionTypeWithExceptionSpec(MQT->getUnderlyingType(), ESI),
-        MQT->getMacroIdentifier());
-
-  // Might have a calling-convention attribute.
-  if (const auto *AT = dyn_cast<AttributedType>(Orig))
-    return getAttributedType(
-        AT->getAttrKind(),
-        getFunctionTypeWithExceptionSpec(AT->getModifiedType(), ESI),
-        getFunctionTypeWithExceptionSpec(AT->getEquivalentType(), ESI));
-
-  // Anything else must be a function type. Rebuild it with the new exception
-  // specification.
-  const auto *Proto = Orig->castAs<FunctionProtoType>();
-  return getFunctionType(
-      Proto->getReturnType(), Proto->getParamTypes(),
-      Proto->getExtProtoInfo().withExceptionSpec(ESI));
+  return adjustType(Orig, [&](QualType Ty) {
+    const auto *Proto = Ty->castAs<FunctionProtoType>();
+    return getFunctionType(Proto->getReturnType(), Proto->getParamTypes(),
+                           Proto->getExtProtoInfo().withExceptionSpec(ESI));
+  });
 }
 
 bool ASTContext::hasSameFunctionTypeIgnoringExceptionSpec(QualType T,
@@ -5165,7 +5200,7 @@ QualType ASTContext::getAttributedType(attr::Kind attrKind,
 }
 
 QualType ASTContext::getBTFTagAttributedType(const BTFTypeTagAttr *BTFAttr,
-                                             QualType Wrapped) {
+                                             QualType Wrapped) const {
   llvm::FoldingSetNodeID ID;
   BTFTagAttributedType::Profile(ID, Wrapped, BTFAttr);
 
diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index b8778f6027894..4fd697ebe4938 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -8,6 +8,7 @@
 
 #include "ByteCodeEmitter.h"
 #include "Context.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "IntegralAP.h"
 #include "Opcode.h"
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index e54b6568d7060..2520fe30547b2 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -9,6 +9,7 @@
 #include "Compiler.h"
 #include "ByteCodeEmitter.h"
 #include "Context.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "Function.h"
 #include "InterpShared.h"
@@ -470,6 +471,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_IntegralToBoolean:
+  case CK_FixedPointToBoolean:
   case CK_BooleanToSignedIntegral:
   case CK_IntegralCast: {
     if (DiscardResult)
@@ -717,6 +719,16 @@ bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
   return this->visitArrayElemInit(1, SubExpr);
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::VisitFixedPointLiteral(const FixedPointLiteral *E) {
+  assert(E->getType()->isFixedPointType());
+  assert(classifyPrim(E) == PT_FixedPoint);
+
+  auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
+  APInt Value = E->getValue();
+  return this->emitConstFixedPoint(FixedPoint(Value, Sem), E);
+}
+
 template <class Emitter>
 bool Compiler<Emitter>::VisitParenExpr(const ParenExpr *E) {
   return this->delegate(E->getSubExpr());
@@ -3685,9 +3697,13 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
     return this->emitNullFnPtr(nullptr, E);
   case PT_MemberPtr:
     return this->emitNullMemberPtr(nullptr, E);
-  case PT_Float: {
+  case PT_Float:
     return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
+  case PT_FixedPoint: {
+    auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
+    return this->emitConstFixedPoint(FixedPoint::Zero(Sem), E);
   }
+    llvm_unreachable("Implement");
   }
   llvm_unreachable("unknown primitive type");
 }
@@ -3798,6 +3814,7 @@ bool Compiler<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
   case PT_Float:
   case PT_IntAP:
   case PT_IntAPS:
+  case PT_FixedPoint:
     llvm_unreachable("Invalid integral type");
     break;
   }
@@ -5293,7 +5310,7 @@ bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
       if (!D->isPrimitive() && !D->isPrimitiveArray()) {
         if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
           return false;
-        if (!this->emitDestruction(D))
+        if (!this->emitDestruction(D, SourceInfo{}))
           return false;
         if (!this->emitPopPtr(SourceInfo{}))
           return false;
@@ -5302,9 +5319,12 @@ bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
   }
 
   for (const Record::Base &Base : llvm::reverse(R->bases())) {
+    if (Base.R->isAnonymousUnion())
+      continue;
+
     if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
       return false;
-    if (!this->emitRecordDestruction(Base.R))
+    if (!this->emitRecordDestruction(Base.R, {}))
       return false;
     if (!this->emitPopPtr(SourceInfo{}))
       return false;
@@ -6145,8 +6165,9 @@ bool Compiler<Emitter>::emitComplexComparison(const Expr *LHS, const Expr *RHS,
 /// on the stack.
 /// Emit destruction of record types (or arrays of record types).
 template <class Emitter>
-bool Compiler<Emitter>::emitRecordDestruction(const Record *R) {
+bool Compiler<Emitter>::emitRecordDestruction(const Record *R, SourceInfo Loc) {
   assert(R);
+  assert(!R->isAnonymousUnion());
   const CXXDestructorDecl *Dtor = R->getDestructor();
   if (!Dtor || Dtor->isTrivial())
     return true;
@@ -6157,15 +6178,16 @@ bool Compiler<Emitter>::emitRecordDestruction(const Record *R) {
     return false;
   assert(DtorFunc->hasThisPointer());
   assert(DtorFunc->getNumParams() == 1);
-  if (!this->emitDupPtr(SourceInfo{}))
+  if (!this->emitDupPtr(Loc))
     return false;
-  return this->emitCall(DtorFunc, 0, SourceInfo{});
+  return this->emitCall(DtorFunc, 0, Loc);
 }
 /// When calling this, we have a pointer of the local-to-destroy
 /// on the stack.
 /// Emit destruction of record types (or arrays of record types).
 template <class Emitter>
-bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc) {
+bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
+                                        SourceInfo Loc) {
   assert(Desc);
   assert(!Desc->isPrimitive());
   assert(!Desc->isPrimitiveArray());
@@ -6189,20 +6211,23 @@ bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc) {
     }
 
     for (ssize_t I = Desc->getNumElems() - 1; I >= 0; --I) {
-      if (!this->emitConstUint64(I, SourceInfo{}))
+      if (!this->emitConstUint64(I, Loc))
         return false;
-      if (!this->emitArrayElemPtrUint64(SourceInfo{}))
+      if (!this->emitArrayElemPtrUint64(Loc))
         return false;
-      if (!this->emitDestruction(ElemDesc))
+      if (!this->emitDestruction(ElemDesc, Loc))
         return false;
-      if (!this->emitPopPtr(SourceInfo{}))
+      if (!this->emitPopPtr(Loc))
         return false;
     }
     return true;
   }
 
   assert(Desc->ElemRecord);
-  return this->emitRecordDestruction(Desc->ElemRecord);
+  if (Desc->ElemRecord->isAnonymousUnion())
+    return true;
+
+  return this->emitRecordDestruction(Desc->ElemRecord, Loc);
 }
 
 namespace clang {
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index 2dfa187713a80..d1911f11603a0 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -125,6 +125,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool VisitIntegerLiteral(const IntegerLiteral *E);
   bool VisitFloatingLiteral(const FloatingLiteral *E);
   bool VisitImaginaryLiteral(const ImaginaryLiteral *E);
+  bool VisitFixedPointLiteral(const FixedPointLiteral *E);
   bool VisitParenExpr(const ParenExpr *E);
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitLogicalBinOp(const BinaryOperator *E);
@@ -364,8 +365,8 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool emitComplexBoolCast(const Expr *E);
   bool emitComplexComparison(const Expr *LHS, const Expr *RHS,
                              const BinaryOperator *E);
-  bool emitRecordDestruction(const Record *R);
-  bool emitDestruction(const Descriptor *Desc);
+  bool emitRecordDestruction(const Record *R, SourceInfo Loc);
+  bool emitDestruction(const Descriptor *Desc, SourceInfo Loc);
   unsigned collectBaseOffset(const QualType BaseType,
                              const QualType DerivedType);
   bool emitLambdaStaticInvokerBody(const CXXMethodDecl *MD);
@@ -540,7 +541,7 @@ template <class Emitter> class LocalScope : public VariableScope<Emitter> {
         if (!this->Ctx->emitGetPtrLocal(Local.Offset, E))
           return false;
 
-        if (!this->Ctx->emitDestruction(Local.Desc))
+        if (!this->Ctx->emitDestruction(Local.Desc, Local.Desc->getLoc()))
           return false;
 
         if (!this->Ctx->emitPopPtr(E))
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index 8661acf536658..9bca8138cd9f6 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -198,6 +198,9 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (const auto *DT = dyn_cast<DecltypeType>(T))
     return classify(DT->getUnderlyingType());
 
+  if (T->isFixedPointType())
+    return PT_FixedPoint;
+
   return std::nullopt;
 }
 
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 05ece907af42f..65ac7a3129aba 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -8,6 +8,7 @@
 
 #include "Descriptor.h"
 #include "Boolean.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
@@ -15,6 +16,7 @@
 #include "Pointer.h"
 #include "PrimType.h"
 #include "Record.h"
+#include "Source.h"
 
 using namespace clang;
 using namespace clang::interp;
@@ -423,6 +425,14 @@ SourceLocation Descriptor::getLocation() const {
   llvm_unreachable("Invalid descriptor type");
 }
 
+SourceInfo Descriptor::getLoc() const {
+  if (const auto *D = Source.dyn_cast<const Decl *>())
+    return SourceInfo(D);
+  if (const auto *E = Source.dyn_cast<const Expr *>())
+    return SourceInfo(E);
+  llvm_unreachable("Invalid descriptor type");
+}
+
 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
 
 InitMap::InitMap(unsigned N)
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 82f90430f7f4e..5460199e0e991 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -21,6 +21,7 @@ namespace clang {
 namespace interp {
 class Block;
 class Record;
+class SourceInfo;
 struct InitMap;
 struct Descriptor;
 enum PrimType : unsigned;
@@ -194,6 +195,7 @@ struct Descriptor final {
   QualType getType() const;
   QualType getElemQualType() const;
   SourceLocation getLocation() const;
+  SourceInfo getLoc() const;
 
   const Decl *asDecl() const { return Source.dyn_cast<const Decl *>(); }
   const Expr *asExpr() const { return Source.dyn_cast<const Expr *>(); }
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index e1051e5c2bbf6..85522ffd32dcc 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -13,6 +13,7 @@
 #include "Boolean.h"
 #include "Context.h"
 #include "EvaluationResult.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "Function.h"
 #include "FunctionPointer.h"
@@ -126,6 +127,8 @@ static const char *primTypeToString(PrimType T) {
     return "FnPtr";
   case PT_MemberPtr:
     return "MemberPtr";
+  case PT_FixedPoint:
+    return "FixedPoint";
   }
   llvm_unreachable("Unhandled PrimType");
 }
diff --git a/clang/lib/AST/ByteCode/FixedPoint.h b/clang/lib/AST/ByteCode/FixedPoint.h
new file mode 100644
index 0000000000000..9dd300c3dbf6d
--- /dev/null
+++ b/clang/lib/AST/ByteCode/FixedPoint.h
@@ -0,0 +1,77 @@
+//===------- FixedPoint.h - Fixedd point types for the VM -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_INTERP_FIXED_POINT_H
+#define LLVM_CLANG_AST_INTERP_FIXED_POINT_H
+
+#include "clang/AST/APValue.h"
+#include "clang/AST/ComparisonCategories.h"
+#include "llvm/ADT/APFixedPoint.h"
+
+namespace clang {
+namespace interp {
+
+using APInt = llvm::APInt;
+using APSInt = llvm::APSInt;
+
+/// Wrapper around fixed point types.
+class FixedPoint final {
+private:
+  llvm::APFixedPoint V;
+  FixedPoint(llvm::APFixedPoint &&V) : V(std::move(V)) {}
+
+public:
+  FixedPoint(APInt V, llvm::FixedPointSemantics Sem) : V(V, Sem) {}
+  // This needs to be default-constructible so llvm::endian::read works.
+  FixedPoint()
+      : V(APInt(0, 0ULL, false),
+          llvm::FixedPointSemantics(0, 0, false, false, false)) {}
+
+  static FixedPoint Zero(llvm::FixedPointSemantics Sem) {
+    return FixedPoint(APInt(Sem.getWidth(), 0ULL, Sem.isSigned()), Sem);
+  }
+
+  operator bool() const { return V.getBoolValue(); }
+  template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
+  explicit operator Ty() const {
+    // FIXME
+    return 0;
+  }
+
+  void print(llvm::raw_ostream &OS) const { OS << V; }
+
+  APValue toAPValue(const ASTContext &) const { return APValue(V); }
+  APSInt toAPSInt(unsigned BitWidth) const { return V.getValue(); }
+
+  unsigned bitWidth() const { return V.getWidth(); }
+  bool isSigned() const { return V.isSigned(); }
+
+  ComparisonCategoryResult compare(const FixedPoint &Other) const {
+    if (Other.V == V)
+      return ComparisonCategoryResult::Equal;
+    return ComparisonCategoryResult::Unordered;
+  }
+
+  static bool neg(const FixedPoint &A, FixedPoint *R) {
+    bool Overflow = false;
+    *R = FixedPoint(A.V.negate(&Overflow));
+    return Overflow;
+  }
+};
+
+inline FixedPoint getSwappedBytes(FixedPoint F) { return F; }
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, FixedPoint F) {
+  F.print(OS);
+  return OS;
+}
+
+} // namespace interp
+} // namespace clang
+
+#endif
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 2f4a05a85753c..798e0f3e96fa0 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1043,6 +1043,25 @@ bool CheckLiteralType(InterpState &S, CodePtr OpPC, const Type *T) {
   return false;
 }
 
+static bool checkConstructor(InterpState &S, CodePtr OpPC, const Function *Func,
+                             const Pointer &ThisPtr) {
+  assert(Func->isConstructor());
+
+  const Descriptor *D = ThisPtr.getFieldDesc();
+
+  // FIXME: I think this case is not 100% correct. E.g. a pointer into a
+  // subobject of a composite array.
+  if (!D->ElemRecord)
+    return true;
+
+  if (D->ElemRecord->getNumVirtualBases() == 0)
+    return true;
+
+  S.FFDiag(S.Current->getLocation(OpPC), diag::note_constexpr_virtual_base)
+      << Func->getParentDecl();
+  return false;
+}
+
 bool CallVar(InterpState &S, CodePtr OpPC, const Function *Func,
              uint32_t VarArgSize) {
   if (Func->hasThisPointer()) {
@@ -1117,6 +1136,9 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
       if (!CheckInvoke(S, OpPC, ThisPtr))
         return cleanup();
     }
+
+    if (Func->isConstructor() && !checkConstructor(S, OpPC, Func, ThisPtr))
+      return false;
   }
 
   if (!CheckCallable(S, OpPC, Func))
@@ -1300,7 +1322,8 @@ bool CheckNewTypeMismatch(InterpState &S, CodePtr OpPC, const Expr *E,
   const auto *NewExpr = cast<CXXNewExpr>(E);
   QualType StorageType = Ptr.getType();
 
-  if (isa_and_nonnull<CXXNewExpr>(Ptr.getFieldDesc()->asExpr())) {
+  if (isa_and_nonnull<CXXNewExpr>(Ptr.getFieldDesc()->asExpr()) &&
+      StorageType->isPointerType()) {
     // FIXME: Are there other cases where this is a problem?
     StorageType = StorageType->getPointeeType();
   }
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index b029399a1554b..79af426f8a913 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -16,6 +16,7 @@
 #include "../ExprConstShared.h"
 #include "Boolean.h"
 #include "DynamicAllocator.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "Function.h"
 #include "FunctionPointer.h"
diff --git a/clang/lib/AST/ByteCode/InterpStack.cpp b/clang/lib/AST/ByteCode/InterpStack.cpp
index ae3721e983741..b183335dd5884 100644
--- a/clang/lib/AST/ByteCode/InterpStack.cpp
+++ b/clang/lib/AST/ByteCode/InterpStack.cpp
@@ -8,6 +8,7 @@
 
 #include "InterpStack.h"
 #include "Boolean.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "Integral.h"
 #include "MemberPointer.h"
diff --git a/clang/lib/AST/ByteCode/InterpStack.h b/clang/lib/AST/ByteCode/InterpStack.h
index 43988bb680d1c..f7b8c386bcc13 100644
--- a/clang/lib/AST/ByteCode/InterpStack.h
+++ b/clang/lib/AST/ByteCode/InterpStack.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_AST_INTERP_INTERPSTACK_H
 #define LLVM_CLANG_AST_INTERP_INTERPSTACK_H
 
+#include "FixedPoint.h"
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
 #include "MemberPointer.h"
@@ -190,6 +191,8 @@ class InterpStack final {
       return PT_IntAP;
     else if constexpr (std::is_same_v<T, MemberPointer>)
       return PT_MemberPtr;
+    else if constexpr (std::is_same_v<T, FixedPoint>)
+      return PT_FixedPoint;
 
     llvm_unreachable("unknown type push()'ed into InterpStack");
   }
diff --git a/clang/lib/AST/ByteCode/MemberPointer.cpp b/clang/lib/AST/ByteCode/MemberPointer.cpp
index 0fe94db97a3c4..dfc8583e464ab 100644
--- a/clang/lib/AST/ByteCode/MemberPointer.cpp
+++ b/clang/lib/AST/ByteCode/MemberPointer.cpp
@@ -79,7 +79,7 @@ APValue MemberPointer::toAPValue(const ASTContext &ASTCtx) const {
   if (hasBase())
     return Base.toAPValue(ASTCtx);
 
-  return APValue(cast<ValueDecl>(getDecl()), /*IsDerivedMember=*/false,
+  return APValue(getDecl(), /*IsDerivedMember=*/false,
                  /*Path=*/{});
 }
 
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 36191f096aeb8..5fdafd1bf8198 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -31,6 +31,7 @@ def Float : Type;
 def Ptr : Type;
 def FnPtr : Type;
 def MemberPtr : Type;
+def FixedPoint : Type;
 
 //===----------------------------------------------------------------------===//
 // Types transferred to the interpreter.
@@ -49,6 +50,7 @@ def ArgIntAP : ArgType { let Name = "IntegralAP<false>"; let AsRef = true; }
 def ArgIntAPS : ArgType { let Name = "IntegralAP<true>"; let AsRef = true; }
 def ArgFloat : ArgType { let Name = "Floating"; let AsRef = true; }
 def ArgBool : ArgType { let Name = "bool"; }
+def ArgFixedPoint : ArgType { let Name = "FixedPoint"; let AsRef = true; }
 
 def ArgFunction : ArgType { let Name = "const Function *"; }
 def ArgRecordDecl : ArgType { let Name = "const RecordDecl *"; }
@@ -104,11 +106,11 @@ def PtrTypeClass : TypeClass {
 }
 
 def NonPtrTypeClass : TypeClass {
-  let Types = !listconcat(IntegerTypeClass.Types, [Bool], [Float]);
+  let Types = !listconcat(IntegerTypeClass.Types, [Bool], [Float], [FixedPoint]);
 }
 
 def AllTypeClass : TypeClass {
-  let Types = !listconcat(AluTypeClass.Types, PtrTypeClass.Types, FloatTypeClass.Types);
+  let Types = !listconcat(AluTypeClass.Types, PtrTypeClass.Types, FloatTypeClass.Types, [FixedPoint]);
 }
 
 def ComparableTypeClass : TypeClass {
@@ -255,6 +257,7 @@ def ConstFloat : ConstOpcode<Float, ArgFloat>;
 def constIntAP : ConstOpcode<IntAP, ArgIntAP>;
 def constIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
 def ConstBool : ConstOpcode<Bool, ArgBool>;
+def ConstFixedPoint : ConstOpcode<FixedPoint, ArgFixedPoint>;
 
 // [] -> [Integer]
 def Zero : Opcode {
@@ -607,7 +610,7 @@ def IsNonNull : Opcode {
 //===----------------------------------------------------------------------===//
 
 def FromCastTypeClass : TypeClass {
-  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool, IntAP, IntAPS];
+  let Types = [Uint8, Sint8, Uint16, Sint16, Uint32, Sint32, Uint64, Sint64, Bool, IntAP, IntAPS, FixedPoint];
 }
 
 def ToCastTypeClass : TypeClass {
diff --git a/clang/lib/AST/ByteCode/PrimType.cpp b/clang/lib/AST/ByteCode/PrimType.cpp
index 3054e67d5c49f..7dbab996416cc 100644
--- a/clang/lib/AST/ByteCode/PrimType.cpp
+++ b/clang/lib/AST/ByteCode/PrimType.cpp
@@ -8,6 +8,7 @@
 
 #include "PrimType.h"
 #include "Boolean.h"
+#include "FixedPoint.h"
 #include "Floating.h"
 #include "FunctionPointer.h"
 #include "IntegralAP.h"
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index bb2f59d86e98d..59c04c4673d93 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -26,6 +26,7 @@ class Boolean;
 class Floating;
 class FunctionPointer;
 class MemberPointer;
+class FixedPoint;
 template <bool Signed> class IntegralAP;
 template <unsigned Bits, bool Signed> class Integral;
 
@@ -42,10 +43,11 @@ enum PrimType : unsigned {
   PT_IntAP = 8,
   PT_IntAPS = 9,
   PT_Bool = 10,
-  PT_Float = 11,
-  PT_Ptr = 12,
-  PT_FnPtr = 13,
-  PT_MemberPtr = 14,
+  PT_FixedPoint = 11,
+  PT_Float = 12,
+  PT_Ptr = 13,
+  PT_FnPtr = 14,
+  PT_MemberPtr = 15,
 };
 
 inline constexpr bool isPtrType(PrimType T) {
@@ -69,7 +71,7 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
   return OS;
 }
 
-constexpr bool isIntegralType(PrimType T) { return T <= PT_Bool; }
+constexpr bool isIntegralType(PrimType T) { return T <= PT_FixedPoint; }
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
@@ -118,6 +120,9 @@ template <> struct PrimConv<PT_FnPtr> {
 template <> struct PrimConv<PT_MemberPtr> {
   using T = MemberPointer;
 };
+template <> struct PrimConv<PT_FixedPoint> {
+  using T = FixedPoint;
+};
 
 /// Returns the size of a primitive type in bytes.
 size_t primSize(PrimType Type);
@@ -163,6 +168,7 @@ static inline bool aligned(const void *P) {
       TYPE_SWITCH_CASE(PT_Ptr, B)                                              \
       TYPE_SWITCH_CASE(PT_FnPtr, B)                                            \
       TYPE_SWITCH_CASE(PT_MemberPtr, B)                                        \
+      TYPE_SWITCH_CASE(PT_FixedPoint, B)                                       \
     }                                                                          \
   } while (0)
 
diff --git a/clang/lib/AST/ByteCode/Record.cpp b/clang/lib/AST/ByteCode/Record.cpp
index ec1b55da347af..0c06bec7e5508 100644
--- a/clang/lib/AST/ByteCode/Record.cpp
+++ b/clang/lib/AST/ByteCode/Record.cpp
@@ -16,7 +16,8 @@ Record::Record(const RecordDecl *Decl, BaseList &&SrcBases,
                FieldList &&SrcFields, VirtualBaseList &&SrcVirtualBases,
                unsigned VirtualSize, unsigned BaseSize)
     : Decl(Decl), Bases(std::move(SrcBases)), Fields(std::move(SrcFields)),
-      BaseSize(BaseSize), VirtualSize(VirtualSize), IsUnion(Decl->isUnion()) {
+      BaseSize(BaseSize), VirtualSize(VirtualSize), IsUnion(Decl->isUnion()),
+      IsAnonymousUnion(IsUnion && Decl->isAnonymousStructOrUnion()) {
   for (Base &V : SrcVirtualBases)
     VirtualBases.push_back({V.Decl, V.Offset + BaseSize, V.Desc, V.R});
 
diff --git a/clang/lib/AST/ByteCode/Record.h b/clang/lib/AST/ByteCode/Record.h
index 83e15b125f77a..7a5c482e4efcc 100644
--- a/clang/lib/AST/ByteCode/Record.h
+++ b/clang/lib/AST/ByteCode/Record.h
@@ -54,6 +54,8 @@ class Record final {
   const std::string getName() const;
   /// Checks if the record is a union.
   bool isUnion() const { return IsUnion; }
+  /// Checks if the record is an anonymous union.
+  bool isAnonymousUnion() const { return IsAnonymousUnion; }
   /// Returns the size of the record.
   unsigned getSize() const { return BaseSize; }
   /// Returns the full size of the record, including records.
@@ -134,6 +136,8 @@ class Record final {
   unsigned VirtualSize;
   /// If this record is a union.
   bool IsUnion;
+  /// If this is an anonymous union.
+  bool IsAnonymousUnion;
 };
 
 } // namespace interp
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 6387e375dda79..960eae36ed1f5 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -54,8 +54,10 @@
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/APFixedPoint.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/SaveAndRestore.h"
 #include "llvm/Support/SipHash.h"
@@ -2061,8 +2063,8 @@ static bool EvaluateIgnoredValue(EvalInfo &Info, const Expr *E) {
   return true;
 }
 
-/// Should this call expression be treated as a no-op?
-static bool IsNoOpCall(const CallExpr *E) {
+/// Should this call expression be treated as forming an opaque constant?
+static bool IsOpaqueConstantCall(const CallExpr *E) {
   unsigned Builtin = E->getBuiltinCallee();
   return (Builtin == Builtin::BI__builtin___CFStringMakeConstantString ||
           Builtin == Builtin::BI__builtin___NSStringMakeConstantString ||
@@ -2070,6 +2072,12 @@ static bool IsNoOpCall(const CallExpr *E) {
           Builtin == Builtin::BI__builtin_function_start);
 }
 
+static bool IsOpaqueConstantCall(const LValue &LVal) {
+  const auto *BaseExpr =
+      llvm::dyn_cast_if_present<CallExpr>(LVal.Base.dyn_cast<const Expr *>());
+  return BaseExpr && IsOpaqueConstantCall(BaseExpr);
+}
+
 static bool IsGlobalLValue(APValue::LValueBase B) {
   // C++11 [expr.const]p3 An address constant expression is a prvalue core
   // constant expression of pointer type that evaluates to...
@@ -2115,7 +2123,7 @@ static bool IsGlobalLValue(APValue::LValueBase B) {
   case Expr::ObjCBoxedExprClass:
     return cast<ObjCBoxedExpr>(E)->isExpressibleAsConstantInitializer();
   case Expr::CallExprClass:
-    return IsNoOpCall(cast<CallExpr>(E));
+    return IsOpaqueConstantCall(cast<CallExpr>(E));
   // For GCC compatibility, &&label has static storage duration.
   case Expr::AddrLabelExprClass:
     return true;
@@ -2142,11 +2150,91 @@ static const ValueDecl *GetLValueBaseDecl(const LValue &LVal) {
   return LVal.Base.dyn_cast<const ValueDecl*>();
 }
 
-static bool IsLiteralLValue(const LValue &Value) {
-  if (Value.getLValueCallIndex())
+// Information about an LValueBase that is some kind of string.
+struct LValueBaseString {
+  std::string ObjCEncodeStorage;
+  StringRef Bytes;
+  int CharWidth;
+};
+
+// Gets the lvalue base of LVal as a string.
+static bool GetLValueBaseAsString(const EvalInfo &Info, const LValue &LVal,
+                                  LValueBaseString &AsString) {
+  const auto *BaseExpr = LVal.Base.dyn_cast<const Expr *>();
+  if (!BaseExpr)
+    return false;
+
+  // For ObjCEncodeExpr, we need to compute and store the string.
+  if (const auto *EE = dyn_cast<ObjCEncodeExpr>(BaseExpr)) {
+    Info.Ctx.getObjCEncodingForType(EE->getEncodedType(),
+                                    AsString.ObjCEncodeStorage);
+    AsString.Bytes = AsString.ObjCEncodeStorage;
+    AsString.CharWidth = 1;
+    return true;
+  }
+
+  // Otherwise, we have a StringLiteral.
+  const auto *Lit = dyn_cast<StringLiteral>(BaseExpr);
+  if (const auto *PE = dyn_cast<PredefinedExpr>(BaseExpr))
+    Lit = PE->getFunctionName();
+
+  if (!Lit)
     return false;
-  const Expr *E = Value.Base.dyn_cast<const Expr*>();
-  return E && !isa<MaterializeTemporaryExpr>(E);
+
+  AsString.Bytes = Lit->getBytes();
+  AsString.CharWidth = Lit->getCharByteWidth();
+  return true;
+}
+
+// Determine whether two string literals potentially overlap. This will be the
+// case if they agree on the values of all the bytes on the overlapping region
+// between them.
+//
+// The overlapping region is the portion of the two string literals that must
+// overlap in memory if the pointers actually point to the same address at
+// runtime. For example, if LHS is "abcdef" + 3 and RHS is "cdef\0gh" + 1 then
+// the overlapping region is "cdef\0", which in this case does agree, so the
+// strings are potentially overlapping. Conversely, for "foobar" + 3 versus
+// "bazbar" + 3, the overlapping region contains all of both strings, so they
+// are not potentially overlapping, even though they agree from the given
+// addresses onwards.
+//
+// See open core issue CWG2765 which is discussing the desired rule here.
+static bool ArePotentiallyOverlappingStringLiterals(const EvalInfo &Info,
+                                                    const LValue &LHS,
+                                                    const LValue &RHS) {
+  LValueBaseString LHSString, RHSString;
+  if (!GetLValueBaseAsString(Info, LHS, LHSString) ||
+      !GetLValueBaseAsString(Info, RHS, RHSString))
+    return false;
+
+  // This is the byte offset to the location of the first character of LHS
+  // within RHS. We don't need to look at the characters of one string that
+  // would appear before the start of the other string if they were merged.
+  CharUnits Offset = RHS.Offset - LHS.Offset;
+  if (Offset.isNegative())
+    LHSString.Bytes = LHSString.Bytes.drop_front(-Offset.getQuantity());
+  else
+    RHSString.Bytes = RHSString.Bytes.drop_front(Offset.getQuantity());
+
+  bool LHSIsLonger = LHSString.Bytes.size() > RHSString.Bytes.size();
+  StringRef Longer = LHSIsLonger ? LHSString.Bytes : RHSString.Bytes;
+  StringRef Shorter = LHSIsLonger ? RHSString.Bytes : LHSString.Bytes;
+  int ShorterCharWidth = (LHSIsLonger ? RHSString : LHSString).CharWidth;
+
+  // The null terminator isn't included in the string data, so check for it
+  // manually. If the longer string doesn't have a null terminator where the
+  // shorter string ends, they aren't potentially overlapping.
+  for (int NullByte : llvm::seq(ShorterCharWidth)) {
+    if (Shorter.size() + NullByte >= Longer.size())
+      break;
+    if (Longer[Shorter.size() + NullByte])
+      return false;
+  }
+
+  // Otherwise, they're potentially overlapping if and only if the overlapping
+  // region is the same.
+  return Shorter == Longer.take_front(Shorter.size());
 }
 
 static bool IsWeakLValue(const LValue &Value) {
@@ -8573,7 +8661,10 @@ class LValueExprEvaluator
   bool VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);
   bool VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
   bool VisitMemberExpr(const MemberExpr *E);
-  bool VisitStringLiteral(const StringLiteral *E) { return Success(E); }
+  bool VisitStringLiteral(const StringLiteral *E) {
+    return Success(APValue::LValueBase(
+        E, 0, Info.getASTContext().getNextStringLiteralVersion()));
+  }
   bool VisitObjCEncodeExpr(const ObjCEncodeExpr *E) { return Success(E); }
   bool VisitCXXTypeidExpr(const CXXTypeidExpr *E);
   bool VisitCXXUuidofExpr(const CXXUuidofExpr *E);
@@ -9639,7 +9730,7 @@ static bool isOneByteCharacterType(QualType T) {
 
 bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
                                                 unsigned BuiltinOp) {
-  if (IsNoOpCall(E))
+  if (IsOpaqueConstantCall(E))
     return Success(E);
 
   switch (BuiltinOp) {
@@ -13889,13 +13980,22 @@ EvaluateComparisonBinaryOperator(EvalInfo &Info, const BinaryOperator *E,
           (!RHSValue.Base && !RHSValue.Offset.isZero()))
         return DiagComparison(diag::note_constexpr_pointer_constant_comparison,
                               !RHSValue.Base);
-      // It's implementation-defined whether distinct literals will have
-      // distinct addresses. In clang, the result of such a comparison is
-      // unspecified, so it is not a constant expression. However, we do know
-      // that the address of a literal will be non-null.
-      if ((IsLiteralLValue(LHSValue) || IsLiteralLValue(RHSValue)) &&
-          LHSValue.Base && RHSValue.Base)
+      // C++2c [intro.object]/10:
+      //   Two objects [...] may have the same address if [...] they are both
+      //   potentially non-unique objects.
+      // C++2c [intro.object]/9:
+      //   An object is potentially non-unique if it is a string literal object,
+      //   the backing array of an initializer list, or a subobject thereof.
+      //
+      // This makes the comparison result unspecified, so it's not a constant
+      // expression.
+      //
+      // TODO: Do we need to handle the initializer list case here?
+      if (ArePotentiallyOverlappingStringLiterals(Info, LHSValue, RHSValue))
         return DiagComparison(diag::note_constexpr_literal_comparison);
+      if (IsOpaqueConstantCall(LHSValue) || IsOpaqueConstantCall(RHSValue))
+        return DiagComparison(diag::note_constexpr_opaque_call_comparison,
+                              !IsOpaqueConstantCall(LHSValue));
       // We can't tell whether weak symbols will end up pointing to the same
       // object.
       if (IsWeakLValue(LHSValue) || IsWeakLValue(RHSValue))
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index 0bd6845085b73..e23362fc7af00 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -136,7 +136,7 @@ void DiagnosticsEngine::Reset(bool soft /*=false*/) {
 
     // Create a DiagState and DiagStatePoint representing diagnostic changes
     // through command-line.
-    DiagStates.emplace_back(*Diags);
+    DiagStates.emplace_back();
     DiagStatesByLoc.appendFirst(&DiagStates.back());
   }
 }
@@ -147,11 +147,8 @@ DiagnosticsEngine::DiagState::getOrAddMapping(diag::kind Diag) {
       DiagMap.insert(std::make_pair(Diag, DiagnosticMapping()));
 
   // Initialize the entry if we added it.
-  if (Result.second) {
-    Result.first->second = DiagIDs.getDefaultMapping(Diag);
-    if (DiagnosticIDs::IsCustomDiag(Diag))
-      DiagIDs.initCustomDiagMapping(Result.first->second, Diag);
-  }
+  if (Result.second)
+    Result.first->second = DiagnosticIDs::getDefaultMapping(Diag);
 
   return Result.first->second;
 }
@@ -293,8 +290,7 @@ void DiagnosticsEngine::DiagStateMap::dump(SourceManager &SrcMgr,
 
       for (auto &Mapping : *Transition.State) {
         StringRef Option =
-            SrcMgr.getDiagnostics().Diags->getWarningOptionForDiag(
-                Mapping.first);
+            DiagnosticIDs::getWarningOptionForDiag(Mapping.first);
         if (!DiagName.empty() && DiagName != Option)
           continue;
 
@@ -338,7 +334,9 @@ void DiagnosticsEngine::PushDiagStatePoint(DiagState *State,
 
 void DiagnosticsEngine::setSeverity(diag::kind Diag, diag::Severity Map,
                                     SourceLocation L) {
-  assert((Diags->isWarningOrExtension(Diag) ||
+  assert(Diag < diag::DIAG_UPPER_LIMIT &&
+         "Can only map builtin diagnostics");
+  assert((Diags->isBuiltinWarningOrExtension(Diag) ||
           (Map == diag::Severity::Fatal || Map == diag::Severity::Error)) &&
          "Cannot map errors into warnings!");
   assert((L.isInvalid() || SourceMgr) && "No SourceMgr for valid location");
@@ -390,8 +388,6 @@ bool DiagnosticsEngine::setSeverityForGroup(diag::Flavor Flavor,
   if (Diags->getDiagnosticsInGroup(Flavor, Group, GroupDiags))
     return true;
 
-  Diags->setGroupSeverity(Group, Map);
-
   // Set the mapping.
   for (diag::kind Diag : GroupDiags)
     setSeverity(Diag, Map, Loc);
@@ -414,7 +410,6 @@ bool DiagnosticsEngine::setDiagnosticGroupWarningAsError(StringRef Group,
   if (Enabled)
     return setSeverityForGroup(diag::Flavor::WarningOrError, Group,
                                diag::Severity::Error);
-  Diags->setGroupSeverity(Group, diag::Severity::Warning);
 
   // Otherwise, we want to set the diagnostic mapping's "no Werror" bit, and
   // potentially downgrade anything already mapped to be a warning.
@@ -446,7 +441,6 @@ bool DiagnosticsEngine::setDiagnosticGroupErrorAsFatal(StringRef Group,
   if (Enabled)
     return setSeverityForGroup(diag::Flavor::WarningOrError, Group,
                                diag::Severity::Fatal);
-  Diags->setGroupSeverity(Group, diag::Severity::Error);
 
   // Otherwise, we want to set the diagnostic mapping's "no Wfatal-errors" bit,
   // and potentially downgrade anything already mapped to be a fatal error.
@@ -479,7 +473,7 @@ void DiagnosticsEngine::setSeverityForAll(diag::Flavor Flavor,
 
   // Set the mapping.
   for (diag::kind Diag : AllDiags)
-    if (Diags->isWarningOrExtension(Diag))
+    if (Diags->isBuiltinWarningOrExtension(Diag))
       setSeverity(Diag, Map, Loc);
 }
 
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index 031d9d7817d1f..d45bb0f392d45 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -102,12 +102,13 @@ const uint32_t StaticDiagInfoDescriptionOffsets[] = {
 #undef DIAG
 };
 
+// Diagnostic classes.
 enum DiagnosticClass {
-  CLASS_NOTE = DiagnosticIDs::CLASS_NOTE,
-  CLASS_REMARK = DiagnosticIDs::CLASS_REMARK,
-  CLASS_WARNING = DiagnosticIDs::CLASS_WARNING,
-  CLASS_EXTENSION = DiagnosticIDs::CLASS_EXTENSION,
-  CLASS_ERROR = DiagnosticIDs::CLASS_ERROR,
+  CLASS_NOTE       = 0x01,
+  CLASS_REMARK     = 0x02,
+  CLASS_WARNING    = 0x03,
+  CLASS_EXTENSION  = 0x04,
+  CLASS_ERROR      = 0x05
 };
 
 struct StaticDiagInfoRec {
@@ -268,60 +269,11 @@ CATEGORY(INSTALLAPI, REFACTORING)
   return Found;
 }
 
-//===----------------------------------------------------------------------===//
-// Custom Diagnostic information
-//===----------------------------------------------------------------------===//
-
-namespace clang {
-namespace diag {
-using CustomDiagDesc = DiagnosticIDs::CustomDiagDesc;
-class CustomDiagInfo {
-  std::vector<CustomDiagDesc> DiagInfo;
-  std::map<CustomDiagDesc, unsigned> DiagIDs;
-  std::map<diag::Group, std::vector<unsigned>> GroupToDiags;
-
-public:
-  /// getDescription - Return the description of the specified custom
-  /// diagnostic.
-  const CustomDiagDesc &getDescription(unsigned DiagID) const {
-    assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
-           "Invalid diagnostic ID");
-    return DiagInfo[DiagID - DIAG_UPPER_LIMIT];
-  }
-
-  unsigned getOrCreateDiagID(DiagnosticIDs::CustomDiagDesc D) {
-    // Check to see if it already exists.
-    std::map<CustomDiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
-    if (I != DiagIDs.end() && I->first == D)
-      return I->second;
-
-    // If not, assign a new ID.
-    unsigned ID = DiagInfo.size() + DIAG_UPPER_LIMIT;
-    DiagIDs.insert(std::make_pair(D, ID));
-    DiagInfo.push_back(D);
-    if (auto Group = D.GetGroup())
-      GroupToDiags[*Group].emplace_back(ID);
-    return ID;
-  }
-
-  ArrayRef<unsigned> getDiagsInGroup(diag::Group G) const {
-    if (auto Diags = GroupToDiags.find(G); Diags != GroupToDiags.end())
-      return Diags->second;
-    return {};
-  }
-};
-
-} // namespace diag
-} // namespace clang
-
-DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) const {
+DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) {
   DiagnosticMapping Info = DiagnosticMapping::Make(
       diag::Severity::Fatal, /*IsUser=*/false, /*IsPragma=*/false);
 
-  if (IsCustomDiag(DiagID)) {
-    Info.setSeverity(
-        CustomDiagInfo->getDescription(DiagID).GetDefaultSeverity());
-  } else if (const StaticDiagInfoRec *StaticInfo = GetDiagInfo(DiagID)) {
+  if (const StaticDiagInfoRec *StaticInfo = GetDiagInfo(DiagID)) {
     Info.setSeverity((diag::Severity)StaticInfo->DefaultSeverity);
 
     if (StaticInfo->WarnNoWerror) {
@@ -334,18 +286,6 @@ DiagnosticMapping DiagnosticIDs::getDefaultMapping(unsigned DiagID) const {
   return Info;
 }
 
-void DiagnosticIDs::initCustomDiagMapping(DiagnosticMapping &Mapping,
-                                          unsigned DiagID) {
-  assert(IsCustomDiag(DiagID));
-  const auto &Diag = CustomDiagInfo->getDescription(DiagID);
-  if (auto Group = Diag.GetGroup()) {
-    GroupInfo GroupInfo = GroupInfos[static_cast<size_t>(*Group)];
-    if (static_cast<diag::Severity>(GroupInfo.Severity) != diag::Severity())
-      Mapping.setSeverity(static_cast<diag::Severity>(GroupInfo.Severity));
-    Mapping.setNoWarningAsError(GroupInfo.HasNoWarningAsError);
-  }
-}
-
 /// getCategoryNumberForDiag - Return the category number that a specified
 /// DiagID belongs to, or 0 if no category.
 unsigned DiagnosticIDs::getCategoryNumberForDiag(unsigned DiagID) {
@@ -403,6 +343,61 @@ bool DiagnosticIDs::isDeferrable(unsigned DiagID) {
   return false;
 }
 
+/// getBuiltinDiagClass - Return the class field of the diagnostic.
+///
+static unsigned getBuiltinDiagClass(unsigned DiagID) {
+  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
+    return Info->Class;
+  return ~0U;
+}
+
+//===----------------------------------------------------------------------===//
+// Custom Diagnostic information
+//===----------------------------------------------------------------------===//
+
+namespace clang {
+  namespace diag {
+    class CustomDiagInfo {
+      typedef std::pair<DiagnosticIDs::Level, std::string> DiagDesc;
+      std::vector<DiagDesc> DiagInfo;
+      std::map<DiagDesc, unsigned> DiagIDs;
+    public:
+
+      /// getDescription - Return the description of the specified custom
+      /// diagnostic.
+      StringRef getDescription(unsigned DiagID) const {
+        assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
+               "Invalid diagnostic ID");
+        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].second;
+      }
+
+      /// getLevel - Return the level of the specified custom diagnostic.
+      DiagnosticIDs::Level getLevel(unsigned DiagID) const {
+        assert(DiagID - DIAG_UPPER_LIMIT < DiagInfo.size() &&
+               "Invalid diagnostic ID");
+        return DiagInfo[DiagID-DIAG_UPPER_LIMIT].first;
+      }
+
+      unsigned getOrCreateDiagID(DiagnosticIDs::Level L, StringRef Message,
+                                 DiagnosticIDs &Diags) {
+        DiagDesc D(L, std::string(Message));
+        // Check to see if it already exists.
+        std::map<DiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
+        if (I != DiagIDs.end() && I->first == D)
+          return I->second;
+
+        // If not, assign a new ID.
+        unsigned ID = DiagInfo.size()+DIAG_UPPER_LIMIT;
+        DiagIDs.insert(std::make_pair(D, ID));
+        DiagInfo.push_back(D);
+        return ID;
+      }
+    };
+
+  } // end diag namespace
+} // end clang namespace
+
+
 //===----------------------------------------------------------------------===//
 // Common Diagnostic implementation
 //===----------------------------------------------------------------------===//
@@ -417,32 +412,38 @@ DiagnosticIDs::~DiagnosticIDs() {}
 ///
 /// \param FormatString A fixed diagnostic format string that will be hashed and
 /// mapped to a unique DiagID.
-unsigned DiagnosticIDs::getCustomDiagID(CustomDiagDesc Diag) {
+unsigned DiagnosticIDs::getCustomDiagID(Level L, StringRef FormatString) {
   if (!CustomDiagInfo)
     CustomDiagInfo.reset(new diag::CustomDiagInfo());
-  return CustomDiagInfo->getOrCreateDiagID(Diag);
+  return CustomDiagInfo->getOrCreateDiagID(L, FormatString, *this);
 }
 
-bool DiagnosticIDs::isWarningOrExtension(unsigned DiagID) const {
-  return DiagID < diag::DIAG_UPPER_LIMIT
-             ? getDiagClass(DiagID) != CLASS_ERROR
-             : CustomDiagInfo->getDescription(DiagID).GetClass() != CLASS_ERROR;
+
+/// isBuiltinWarningOrExtension - Return true if the unmapped diagnostic
+/// level of the specified diagnostic ID is a Warning or Extension.
+/// This only works on builtin diagnostics, not custom ones, and is not legal to
+/// call on NOTEs.
+bool DiagnosticIDs::isBuiltinWarningOrExtension(unsigned DiagID) {
+  return DiagID < diag::DIAG_UPPER_LIMIT &&
+         getBuiltinDiagClass(DiagID) != CLASS_ERROR;
 }
 
 /// Determine whether the given built-in diagnostic ID is a
 /// Note.
-bool DiagnosticIDs::isNote(unsigned DiagID) const {
-  return DiagID < diag::DIAG_UPPER_LIMIT && getDiagClass(DiagID) == CLASS_NOTE;
+bool DiagnosticIDs::isBuiltinNote(unsigned DiagID) {
+  return DiagID < diag::DIAG_UPPER_LIMIT &&
+    getBuiltinDiagClass(DiagID) == CLASS_NOTE;
 }
 
-/// isExtensionDiag - Determine whether the given built-in diagnostic
+/// isBuiltinExtensionDiag - Determine whether the given built-in diagnostic
 /// ID is for an extension of some sort.  This also returns EnabledByDefault,
 /// which is set to indicate whether the diagnostic is ignored by default (in
 /// which case -pedantic enables it) or treated as a warning/error by default.
 ///
-bool DiagnosticIDs::isExtensionDiag(unsigned DiagID,
-                                    bool &EnabledByDefault) const {
-  if (IsCustomDiag(DiagID) || getDiagClass(DiagID) != CLASS_EXTENSION)
+bool DiagnosticIDs::isBuiltinExtensionDiag(unsigned DiagID,
+                                        bool &EnabledByDefault) {
+  if (DiagID >= diag::DIAG_UPPER_LIMIT ||
+      getBuiltinDiagClass(DiagID) != CLASS_EXTENSION)
     return false;
 
   EnabledByDefault =
@@ -450,7 +451,10 @@ bool DiagnosticIDs::isExtensionDiag(unsigned DiagID,
   return true;
 }
 
-bool DiagnosticIDs::isDefaultMappingAsError(unsigned DiagID) const {
+bool DiagnosticIDs::isDefaultMappingAsError(unsigned DiagID) {
+  if (DiagID >= diag::DIAG_UPPER_LIMIT)
+    return false;
+
   return getDefaultMapping(DiagID).getSeverity() >= diag::Severity::Error;
 }
 
@@ -460,7 +464,7 @@ StringRef DiagnosticIDs::getDescription(unsigned DiagID) const {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return Info->getDescription();
   assert(CustomDiagInfo && "Invalid CustomDiagInfo");
-  return CustomDiagInfo->getDescription(DiagID).GetDescription();
+  return CustomDiagInfo->getDescription(DiagID);
 }
 
 static DiagnosticIDs::Level toLevel(diag::Severity SV) {
@@ -485,7 +489,13 @@ static DiagnosticIDs::Level toLevel(diag::Severity SV) {
 DiagnosticIDs::Level
 DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
                                   const DiagnosticsEngine &Diag) const {
-  unsigned DiagClass = getDiagClass(DiagID);
+  // Handle custom diagnostics, which cannot be mapped.
+  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
+    assert(CustomDiagInfo && "Invalid CustomDiagInfo");
+    return CustomDiagInfo->getLevel(DiagID);
+  }
+
+  unsigned DiagClass = getBuiltinDiagClass(DiagID);
   if (DiagClass == CLASS_NOTE) return DiagnosticIDs::Note;
   return toLevel(getDiagnosticSeverity(DiagID, Loc, Diag));
 }
@@ -499,8 +509,7 @@ DiagnosticIDs::getDiagnosticLevel(unsigned DiagID, SourceLocation Loc,
 diag::Severity
 DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
                                      const DiagnosticsEngine &Diag) const {
-  bool IsCustomDiag = DiagnosticIDs::IsCustomDiag(DiagID);
-  assert(getDiagClass(DiagID) != CLASS_NOTE);
+  assert(getBuiltinDiagClass(DiagID) != CLASS_NOTE);
 
   // Specific non-error diagnostics may be mapped to various levels from ignored
   // to error.  Errors can only be mapped to fatal.
@@ -508,7 +517,7 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
 
   // Get the mapping information, or compute it lazily.
   DiagnosticsEngine::DiagState *State = Diag.GetDiagStateForLoc(Loc);
-  DiagnosticMapping Mapping = State->getOrAddMapping((diag::kind)DiagID);
+  DiagnosticMapping &Mapping = State->getOrAddMapping((diag::kind)DiagID);
 
   // TODO: Can a null severity really get here?
   if (Mapping.getSeverity() != diag::Severity())
@@ -516,15 +525,14 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
 
   // Upgrade ignored diagnostics if -Weverything is enabled.
   if (State->EnableAllWarnings && Result == diag::Severity::Ignored &&
-      !Mapping.isUser() &&
-      (IsCustomDiag || getDiagClass(DiagID) != CLASS_REMARK))
+      !Mapping.isUser() && getBuiltinDiagClass(DiagID) != CLASS_REMARK)
     Result = diag::Severity::Warning;
 
   // Ignore -pedantic diagnostics inside __extension__ blocks.
   // (The diagnostics controlled by -pedantic are the extension diagnostics
   // that are not enabled by default.)
   bool EnabledByDefault = false;
-  bool IsExtensionDiag = isExtensionDiag(DiagID, EnabledByDefault);
+  bool IsExtensionDiag = isBuiltinExtensionDiag(DiagID, EnabledByDefault);
   if (Diag.AllExtensionsSilenced && IsExtensionDiag && !EnabledByDefault)
     return diag::Severity::Ignored;
 
@@ -542,12 +550,10 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
   // as well as disabling all messages which are currently mapped to Warning
   // (whether by default or downgraded from Error via e.g. -Wno-error or #pragma
   // diagnostic.)
-  // FIXME: Should -w be ignored for custom warnings without a group?
   if (State->IgnoreAllWarnings) {
-    if ((!IsCustomDiag || CustomDiagInfo->getDescription(DiagID).GetGroup()) &&
-        (Result == diag::Severity::Warning ||
-         (Result >= diag::Severity::Error &&
-          !isDefaultMappingAsError((diag::kind)DiagID))))
+    if (Result == diag::Severity::Warning ||
+        (Result >= diag::Severity::Error &&
+         !isDefaultMappingAsError((diag::kind)DiagID)))
       return diag::Severity::Ignored;
   }
 
@@ -569,10 +575,9 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
       DiagID != diag::fatal_too_many_errors && Diag.FatalsAsError)
     Result = diag::Severity::Error;
 
+  // Custom diagnostics always are emitted in system headers.
   bool ShowInSystemHeader =
-      IsCustomDiag
-          ? CustomDiagInfo->getDescription(DiagID).ShouldShowInSystemHeader()
-          : !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemHeader;
+      !GetDiagInfo(DiagID) || GetDiagInfo(DiagID)->WarnShowInSystemHeader;
 
   // If we are in a system header, we ignore it. We look at the diagnostic class
   // because we also want to ignore extensions and warnings in -Werror and
@@ -592,15 +597,6 @@ DiagnosticIDs::getDiagnosticSeverity(unsigned DiagID, SourceLocation Loc,
   return Result;
 }
 
-DiagnosticIDs::Class DiagnosticIDs::getDiagClass(unsigned DiagID) const {
-  if (IsCustomDiag(DiagID))
-    return Class(CustomDiagInfo->getDescription(DiagID).GetClass());
-
-  if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
-    return Class(Info->Class);
-  return CLASS_INVALID;
-}
-
 #define GET_DIAG_ARRAYS
 #include "clang/Basic/DiagnosticGroups.inc"
 #undef GET_DIAG_ARRAYS
@@ -646,12 +642,7 @@ DiagnosticIDs::getGroupForWarningOption(StringRef Name) {
   return static_cast<diag::Group>(Found - OptionTable);
 }
 
-std::optional<diag::Group>
-DiagnosticIDs::getGroupForDiag(unsigned DiagID) const {
-  if (IsCustomDiag(DiagID)) {
-    assert(CustomDiagInfo);
-    return CustomDiagInfo->getDescription(DiagID).GetGroup();
-  }
+std::optional<diag::Group> DiagnosticIDs::getGroupForDiag(unsigned DiagID) {
   if (const StaticDiagInfoRec *Info = GetDiagInfo(DiagID))
     return static_cast<diag::Group>(Info->getOptionGroupIndex());
   return std::nullopt;
@@ -682,8 +673,7 @@ std::vector<std::string> DiagnosticIDs::getDiagnosticFlags() {
 /// were filtered out due to having the wrong flavor.
 static bool getDiagnosticsInGroup(diag::Flavor Flavor,
                                   const WarningOption *Group,
-                                  SmallVectorImpl<diag::kind> &Diags,
-                                  diag::CustomDiagInfo *CustomDiagInfo) {
+                                  SmallVectorImpl<diag::kind> &Diags) {
   // An empty group is considered to be a warning group: we have empty groups
   // for GCC compatibility, and GCC does not have remarks.
   if (!Group->Members && !Group->SubGroups)
@@ -702,14 +692,9 @@ static bool getDiagnosticsInGroup(diag::Flavor Flavor,
 
   // Add the members of the subgroups.
   const int16_t *SubGroups = DiagSubGroups + Group->SubGroups;
-  for (; *SubGroups != (int16_t)-1; ++SubGroups) {
-    if (CustomDiagInfo)
-      llvm::copy(
-          CustomDiagInfo->getDiagsInGroup(static_cast<diag::Group>(*SubGroups)),
-          std::back_inserter(Diags));
+  for (; *SubGroups != (int16_t)-1; ++SubGroups)
     NotFound &= getDiagnosticsInGroup(Flavor, &OptionTable[(short)*SubGroups],
-                                      Diags, CustomDiagInfo);
-  }
+                                      Diags);
 
   return NotFound;
 }
@@ -717,49 +702,12 @@ static bool getDiagnosticsInGroup(diag::Flavor Flavor,
 bool
 DiagnosticIDs::getDiagnosticsInGroup(diag::Flavor Flavor, StringRef Group,
                                      SmallVectorImpl<diag::kind> &Diags) const {
-  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
-    if (CustomDiagInfo)
-      llvm::copy(CustomDiagInfo->getDiagsInGroup(*G),
-                 std::back_inserter(Diags));
-    return ::getDiagnosticsInGroup(Flavor,
-                                   &OptionTable[static_cast<unsigned>(*G)],
-                                   Diags, CustomDiagInfo.get());
-  }
+  if (std::optional<diag::Group> G = getGroupForWarningOption(Group))
+    return ::getDiagnosticsInGroup(
+        Flavor, &OptionTable[static_cast<unsigned>(*G)], Diags);
   return true;
 }
 
-template <class Func>
-static void forEachSubGroupImpl(const WarningOption *Group, Func func) {
-  for (const int16_t *SubGroups = DiagSubGroups + Group->SubGroups;
-       *SubGroups != -1; ++SubGroups) {
-    func(static_cast<size_t>(*SubGroups));
-    forEachSubGroupImpl(&OptionTable[*SubGroups], std::move(func));
-  }
-}
-
-template <class Func>
-static void forEachSubGroup(diag::Group Group, Func func) {
-  const WarningOption *WarningOpt = &OptionTable[static_cast<size_t>(Group)];
-  func(static_cast<size_t>(Group));
-  ::forEachSubGroupImpl(WarningOpt, std::move(func));
-}
-
-void DiagnosticIDs::setGroupSeverity(StringRef Group, diag::Severity Sev) {
-  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
-    ::forEachSubGroup(*G, [&](size_t SubGroup) {
-      GroupInfos[SubGroup].Severity = static_cast<unsigned>(Sev);
-    });
-  }
-}
-
-void DiagnosticIDs::setGroupNoWarningsAsError(StringRef Group, bool Val) {
-  if (std::optional<diag::Group> G = getGroupForWarningOption(Group)) {
-    ::forEachSubGroup(*G, [&](size_t SubGroup) {
-      GroupInfos[static_cast<size_t>(*G)].HasNoWarningAsError = Val;
-    });
-  }
-}
-
 void DiagnosticIDs::getAllDiagnostics(diag::Flavor Flavor,
                                       std::vector<diag::kind> &Diags) {
   for (unsigned i = 0; i != StaticDiagInfoSize; ++i)
@@ -782,7 +730,7 @@ StringRef DiagnosticIDs::getNearestOption(diag::Flavor Flavor,
 
     // Don't suggest groups that are not of this kind.
     llvm::SmallVector<diag::kind, 8> Diags;
-    if (::getDiagnosticsInGroup(Flavor, &O, Diags, nullptr) || Diags.empty())
+    if (::getDiagnosticsInGroup(Flavor, &O, Diags) || Diags.empty())
       continue;
 
     if (Distance == BestDistance) {
@@ -896,8 +844,14 @@ void DiagnosticIDs::EmitDiag(DiagnosticsEngine &Diag,
 }
 
 bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
+  if (DiagID >= diag::DIAG_UPPER_LIMIT) {
+    assert(CustomDiagInfo && "Invalid CustomDiagInfo");
+    // Custom diagnostics.
+    return CustomDiagInfo->getLevel(DiagID) >= DiagnosticIDs::Error;
+  }
+
   // Only errors may be unrecoverable.
-  if (getDiagClass(DiagID) < CLASS_ERROR)
+  if (getBuiltinDiagClass(DiagID) < CLASS_ERROR)
     return false;
 
   if (DiagID == diag::err_unavailable ||
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 29f5cd14e46e1..9219533382109 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -198,6 +198,22 @@ TargetInfo::checkCFProtectionBranchSupported(DiagnosticsEngine &Diags) const {
   return false;
 }
 
+CFBranchLabelSchemeKind TargetInfo::getDefaultCFBranchLabelScheme() const {
+  // if this hook is called, the target should override it to return a
+  // non-default scheme
+  llvm::report_fatal_error("not implemented");
+}
+
+bool TargetInfo::checkCFBranchLabelSchemeSupported(
+    const CFBranchLabelSchemeKind Scheme, DiagnosticsEngine &Diags) const {
+  if (Scheme != CFBranchLabelSchemeKind::Default)
+    Diags.Report(diag::err_opt_not_valid_on_target)
+        << (Twine("mcf-branch-label-scheme=") +
+            getCFBranchLabelSchemeFlagVal(Scheme))
+               .str();
+  return false;
+}
+
 bool
 TargetInfo::checkCFProtectionReturnSupported(DiagnosticsEngine &Diags) const {
   Diags.Report(diag::err_opt_not_valid_on_target) << "cf-protection=return";
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 351ef21e197c4..bf40edb8683b3 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -134,6 +134,28 @@ class RISCVTargetInfo : public TargetInfo {
 
   bool validateGlobalRegisterVariable(StringRef RegName, unsigned RegSize,
                                       bool &HasSizeMismatch) const override;
+
+  bool checkCFProtectionBranchSupported(DiagnosticsEngine &) const override {
+    // Always generate Zicfilp lpad insns
+    // Non-zicfilp CPUs would read them as NOP
+    return true;
+  }
+
+  CFBranchLabelSchemeKind getDefaultCFBranchLabelScheme() const override {
+    return CFBranchLabelSchemeKind::FuncSig;
+  }
+
+  bool
+  checkCFBranchLabelSchemeSupported(const CFBranchLabelSchemeKind Scheme,
+                                    DiagnosticsEngine &Diags) const override {
+    switch (Scheme) {
+    case CFBranchLabelSchemeKind::Default:
+    case CFBranchLabelSchemeKind::Unlabeled:
+    case CFBranchLabelSchemeKind::FuncSig:
+      return true;
+    }
+    return TargetInfo::checkCFBranchLabelSchemeSupported(Scheme, Diags);
+  }
 };
 class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
 public:
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 8a26db7971cba..cc79562de2871 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -386,6 +386,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
     PointerWidth = PointerAlign = 64;
     SizeType = TargetInfo::UnsignedLong;
     PtrDiffType = IntPtrType = TargetInfo::SignedLong;
+    AddrSpaceMap = &SPIRDefIsGenMap;
 
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-"
                     "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1-P4-A0");
@@ -418,6 +419,10 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
 
   void setAuxTarget(const TargetInfo *Aux) override;
 
+  void adjust(DiagnosticsEngine &Diags, LangOptions &Opts) override {
+    TargetInfo::adjust(Diags, Opts);
+  }
+
   bool hasInt128Type() const override { return TargetInfo::hasInt128Type(); }
 };
 
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 916c92adb8930..62c6a57e8b7c8 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -429,7 +429,6 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.BBSections =
       llvm::StringSwitch<llvm::BasicBlockSection>(CodeGenOpts.BBSections)
           .Case("all", llvm::BasicBlockSection::All)
-          .Case("labels", llvm::BasicBlockSection::Labels)
           .StartsWith("list=", llvm::BasicBlockSection::List)
           .Case("none", llvm::BasicBlockSection::None)
           .Default(llvm::BasicBlockSection::None);
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 249aead33ad73..9033cd1ccd781 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -13573,6 +13573,14 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
     Int = Intrinsic::aarch64_neon_famax;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "famax");
   }
+  case NEON::BI__builtin_neon_vscale_f16:
+  case NEON::BI__builtin_neon_vscaleq_f16:
+  case NEON::BI__builtin_neon_vscale_f32:
+  case NEON::BI__builtin_neon_vscaleq_f32:
+  case NEON::BI__builtin_neon_vscaleq_f64: {
+    Int = Intrinsic::aarch64_neon_fp8_fscale;
+    return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "fscale");
+  }
   }
 }
 
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 9166db4c74128..df4994ba9af6e 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -122,7 +122,7 @@ RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
       Builder.SetInsertPoint(getPostAllocaInsertPoint());
     V = getTargetHooks().performAddrSpaceCast(
         *this, V, getASTAllocaAddressSpace(), LangAS::Default,
-        Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
+        Builder.getPtrTy(DestAddrSpace), /*non-null*/ true);
   }
 
   return RawAddress(V, Ty, Align, KnownNonNull);
@@ -469,7 +469,8 @@ static RawAddress createReferenceTemporary(CodeGenFunction &CGF,
         if (AS != LangAS::Default)
           C = TCG.performAddrSpaceCast(
               CGF.CGM, GV, AS, LangAS::Default,
-              GV->getValueType()->getPointerTo(
+              llvm::PointerType::get(
+                  CGF.getLLVMContext(),
                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
         // FIXME: Should we put the new global into a COMDAT?
         return RawAddress(C, GV->getValueType(), alignment);
@@ -3207,7 +3208,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
 
     if (AS != T.getAddressSpace()) {
       auto TargetAS = getContext().getTargetAddressSpace(T.getAddressSpace());
-      auto PtrTy = ATPO.getElementType()->getPointerTo(TargetAS);
+      auto PtrTy = llvm::PointerType::get(CGM.getLLVMContext(), TargetAS);
       auto ASC = getTargetHooks().performAddrSpaceCast(
           CGM, ATPO.getPointer(), AS, T.getAddressSpace(), PtrTy);
       ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());
@@ -3835,9 +3836,7 @@ void CodeGenFunction::EmitCfiCheckFail() {
       llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
 
   llvm::Value *V = Builder.CreateConstGEP2_32(
-      CfiCheckFailDataTy,
-      Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
-      0);
+      CfiCheckFailDataTy, Builder.CreatePointerCast(Data, UnqualPtrTy), 0, 0);
 
   Address CheckKindAddr(V, Int8Ty, getIntAlign());
   llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
@@ -6115,36 +6114,6 @@ RValue CodeGenFunction::EmitCall(QualType CalleeType,
   if (ResolvedFnInfo)
     *ResolvedFnInfo = &FnInfo;
 
-  // C99 6.5.2.2p6:
-  //   If the expression that denotes the called function has a type
-  //   that does not include a prototype, [the default argument
-  //   promotions are performed]. If the number of arguments does not
-  //   equal the number of parameters, the behavior is undefined. If
-  //   the function is defined with a type that includes a prototype,
-  //   and either the prototype ends with an ellipsis (, ...) or the
-  //   types of the arguments after promotion are not compatible with
-  //   the types of the parameters, the behavior is undefined. If the
-  //   function is defined with a type that does not include a
-  //   prototype, and the types of the arguments after promotion are
-  //   not compatible with those of the parameters after promotion,
-  //   the behavior is undefined [except in some trivial cases].
-  // That is, in the general case, we should assume that a call
-  // through an unprototyped function type works like a *non-variadic*
-  // call.  The way we make this work is to cast to the exact type
-  // of the promoted arguments.
-  //
-  // Chain calls use this same code path to add the invisible chain parameter
-  // to the function type.
-  if (isa<FunctionNoProtoType>(FnType) || Chain) {
-    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
-    int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
-    CalleeTy = CalleeTy->getPointerTo(AS);
-
-    llvm::Value *CalleePtr = Callee.getFunctionPointer();
-    CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
-    Callee.setFunctionPointer(CalleePtr);
-  }
-
   // HIP function pointer contains kernel handle when it is used in triple
   // chevron. The kernel stub needs to be loaded from kernel handle and used
   // as callee.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index d53d47979f29f..2381fa93e23fe 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1164,6 +1164,16 @@ void CodeGenModule::Release() {
     // Indicate that we want to instrument branch control flow protection.
     getModule().addModuleFlag(llvm::Module::Min, "cf-protection-branch",
                               1);
+
+    auto Scheme = CodeGenOpts.getCFBranchLabelScheme();
+    if (Target.checkCFBranchLabelSchemeSupported(Scheme, getDiags())) {
+      if (Scheme == CFBranchLabelSchemeKind::Default)
+        Scheme = Target.getDefaultCFBranchLabelScheme();
+      getModule().addModuleFlag(
+          llvm::Module::Error, "cf-branch-label-scheme",
+          llvm::MDString::get(getLLVMContext(),
+                              getCFBranchLabelSchemeFlagVal(Scheme)));
+    }
   }
 
   if (CodeGenOpts.FunctionReturnThunks)
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index d5e8e4f7a5916..7dd5c518e7149 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -58,6 +58,8 @@ class SPIRVTargetCodeGenInfo : public CommonSPIRTargetCodeGenInfo {
   SPIRVTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
       : CommonSPIRTargetCodeGenInfo(std::make_unique<SPIRVABIInfo>(CGT)) {}
   void setCUDAKernelCallingConvention(const FunctionType *&FT) const override;
+  LangAS getGlobalVarAddressSpace(CodeGenModule &CGM,
+                                  const VarDecl *D) const override;
   llvm::SyncScope::ID getLLVMSyncScopeID(const LangOptions &LangOpts,
                                          SyncScope Scope,
                                          llvm::AtomicOrdering Ordering,
@@ -217,6 +219,28 @@ void SPIRVTargetCodeGenInfo::setCUDAKernelCallingConvention(
   }
 }
 
+LangAS
+SPIRVTargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
+                                                 const VarDecl *D) const {
+  assert(!CGM.getLangOpts().OpenCL &&
+         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
+         "Address space agnostic languages only");
+  // If we're here it means that we're using the SPIRDefIsGen ASMap, hence for
+  // the global AS we can rely on either cuda_device or sycl_global to be
+  // correct; however, since this is not a CUDA Device context, we use
+  // sycl_global to prevent confusion with the assertion.
+  LangAS DefaultGlobalAS = getLangASFromTargetAS(
+      CGM.getContext().getTargetAddressSpace(LangAS::sycl_global));
+  if (!D)
+    return DefaultGlobalAS;
+
+  LangAS AddrSpace = D->getType().getAddressSpace();
+  if (AddrSpace != LangAS::Default)
+    return AddrSpace;
+
+  return DefaultGlobalAS;
+}
+
 llvm::SyncScope::ID
 SPIRVTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &, SyncScope Scope,
                                            llvm::AtomicOrdering,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a883ba2a25412..b9987288d82d1 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -4424,21 +4424,7 @@ static void RenderDiagnosticsOptions(const Driver &D, const ArgList &Args,
       CmdArgs.push_back("-fno-diagnostics-show-note-include-stack");
   }
 
-  // Color diagnostics are parsed by the driver directly from argv and later
-  // re-parsed to construct this job; claim any possible color diagnostic here
-  // to avoid warn_drv_unused_argument and diagnose bad
-  // OPT_fdiagnostics_color_EQ values.
-  Args.getLastArg(options::OPT_fcolor_diagnostics,
-                  options::OPT_fno_color_diagnostics);
-  if (const Arg *A = Args.getLastArg(options::OPT_fdiagnostics_color_EQ)) {
-    StringRef Value(A->getValue());
-    if (Value != "always" && Value != "never" && Value != "auto")
-      D.Diag(diag::err_drv_invalid_argument_to_option)
-          << Value << A->getOption().getName();
-  }
-
-  if (D.getDiags().getDiagnosticOptions().ShowColors)
-    CmdArgs.push_back("-fcolor-diagnostics");
+  handleColorDiagnosticsArgs(D, Args, CmdArgs);
 
   if (Args.hasArg(options::OPT_fansi_escape_codes))
     CmdArgs.push_back("-fansi-escape-codes");
@@ -6228,9 +6214,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_fbasic_block_sections_EQ)) {
     StringRef Val = A->getValue();
-    if (Triple.isX86() && Triple.isOSBinFormatELF()) {
-      if (Val != "all" && Val != "labels" && Val != "none" &&
-          !Val.starts_with("list="))
+    if (Val == "labels") {
+      D.Diag(diag::warn_drv_deprecated_arg)
+          << A->getAsString(Args) << /*hasReplacement=*/true
+          << "-fbasic-block-address-map";
+      CmdArgs.push_back("-fbasic-block-address-map");
+    } else if (Triple.isX86() && Triple.isOSBinFormatELF()) {
+      if (Val != "all" && Val != "none" && !Val.starts_with("list="))
         D.Diag(diag::err_drv_invalid_value)
             << A->getAsString(Args) << A->getValue();
       else
@@ -7012,6 +7002,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A = Args.getLastArg(options::OPT_fcf_protection_EQ)) {
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-fcf-protection=") + A->getValue()));
+
+    if (Arg *SA = Args.getLastArg(options::OPT_mcf_branch_label_scheme_EQ))
+      CmdArgs.push_back(Args.MakeArgString(Twine("-mcf-branch-label-scheme=") +
+                                           SA->getValue()));
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_mfunction_return_EQ))
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 043d9e4876443..0c6a585c3acff 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2960,3 +2960,22 @@ void tools::addMCModel(const Driver &D, const llvm::opt::ArgList &Args,
     }
   }
 }
+
+void tools::handleColorDiagnosticsArgs(const Driver &D, const ArgList &Args,
+                                       ArgStringList &CmdArgs) {
+  // Color diagnostics are parsed by the driver directly from argv and later
+  // re-parsed to construct this job; claim any possible color diagnostic here
+  // to avoid warn_drv_unused_argument and diagnose bad
+  // OPT_fdiagnostics_color_EQ values.
+  Args.getLastArg(options::OPT_fcolor_diagnostics,
+                  options::OPT_fno_color_diagnostics);
+  if (const Arg *A = Args.getLastArg(options::OPT_fdiagnostics_color_EQ)) {
+    StringRef Value(A->getValue());
+    if (Value != "always" && Value != "never" && Value != "auto")
+      D.Diag(diag::err_drv_invalid_argument_to_option)
+          << Value << A->getOption().getName();
+  }
+
+  if (D.getDiags().getDiagnosticOptions().ShowColors)
+    CmdArgs.push_back("-fcolor-diagnostics");
+}
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index 8695d3fe5b55b..eff21b210b424 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -233,6 +233,10 @@ void addMCModel(const Driver &D, const llvm::opt::ArgList &Args,
                 const llvm::Reloc::Model &RelocationModel,
                 llvm::opt::ArgStringList &CmdArgs);
 
+/// Handle the -f{no}-color-diagnostics and -f{no}-diagnostics-colors options.
+void handleColorDiagnosticsArgs(const Driver &D, const llvm::opt::ArgList &Args,
+                                llvm::opt::ArgStringList &CmdArgs);
+
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 6ce79d27e98c4..98350690f8d20 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -727,13 +727,7 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
   addFortranDialectOptions(Args, CmdArgs);
 
-  // Color diagnostics are parsed by the driver directly from argv and later
-  // re-parsed to construct this job; claim any possible color diagnostic here
-  // to avoid warn_drv_unused_argument.
-  Args.getLastArg(options::OPT_fcolor_diagnostics,
-                  options::OPT_fno_color_diagnostics);
-  if (Diags.getDiagnosticOptions().ShowColors)
-    CmdArgs.push_back("-fcolor-diagnostics");
+  handleColorDiagnosticsArgs(D, Args, CmdArgs);
 
   // LTO mode is parsed by the Clang driver library.
   LTOKind LTOMode = D.getLTOMode();
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index efd852593468a..6c09843a7146f 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1688,6 +1688,18 @@ void CompilerInvocationBase::GenerateCodeGenArgs(const CodeGenOptions &Opts,
   else if (Opts.CFProtectionBranch)
     GenerateArg(Consumer, OPT_fcf_protection_EQ, "branch");
 
+  if (Opts.CFProtectionBranch) {
+    switch (Opts.getCFBranchLabelScheme()) {
+    case CFBranchLabelSchemeKind::Default:
+      break;
+#define CF_BRANCH_LABEL_SCHEME(Kind, FlagVal)                                  \
+  case CFBranchLabelSchemeKind::Kind:                                          \
+    GenerateArg(Consumer, OPT_mcf_branch_label_scheme_EQ, #FlagVal);           \
+    break;
+#include "clang/Basic/CFProtectionOptions.def"
+    }
+  }
+
   if (Opts.FunctionReturnThunks)
     GenerateArg(Consumer, OPT_mfunction_return_EQ, "thunk-extern");
 
@@ -2022,6 +2034,22 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
   }
 
+  if (Opts.CFProtectionBranch && T.isRISCV()) {
+    if (const Arg *A = Args.getLastArg(OPT_mcf_branch_label_scheme_EQ)) {
+      const auto Scheme =
+          llvm::StringSwitch<CFBranchLabelSchemeKind>(A->getValue())
+#define CF_BRANCH_LABEL_SCHEME(Kind, FlagVal)                                  \
+  .Case(#FlagVal, CFBranchLabelSchemeKind::Kind)
+#include "clang/Basic/CFProtectionOptions.def"
+              .Default(CFBranchLabelSchemeKind::Default);
+      if (Scheme != CFBranchLabelSchemeKind::Default)
+        Opts.setCFBranchLabelScheme(Scheme);
+      else
+        Diags.Report(diag::err_drv_invalid_value)
+            << A->getAsString(Args) << A->getValue();
+    }
+  }
+
   if (const Arg *A = Args.getLastArg(OPT_mfunction_return_EQ)) {
     auto Val = llvm::StringSwitch<llvm::FunctionReturnThunksKind>(A->getValue())
                    .Case("keep", llvm::FunctionReturnThunksKind::Keep)
@@ -3952,6 +3980,18 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
     }
   }
 
+  if (Opts.CFProtectionBranch) {
+    if (const Arg *A = Args.getLastArg(OPT_mcf_branch_label_scheme_EQ)) {
+      const auto Scheme =
+          llvm::StringSwitch<CFBranchLabelSchemeKind>(A->getValue())
+#define CF_BRANCH_LABEL_SCHEME(Kind, FlagVal)                                  \
+  .Case(#FlagVal, CFBranchLabelSchemeKind::Kind)
+#include "clang/Basic/CFProtectionOptions.def"
+              .Default(CFBranchLabelSchemeKind::Default);
+      Opts.setCFBranchLabelScheme(Scheme);
+    }
+  }
+
   if ((Args.hasArg(OPT_fsycl_is_device) || Args.hasArg(OPT_fsycl_is_host)) &&
       !Args.hasArg(OPT_sycl_std_EQ)) {
     // If the user supplied -fsycl-is-device or -fsycl-is-host, but failed to
diff --git a/clang/lib/Frontend/LogDiagnosticPrinter.cpp b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
index 4e963af837f01..469d1c22633aa 100644
--- a/clang/lib/Frontend/LogDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/LogDiagnosticPrinter.cpp
@@ -129,8 +129,7 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
   DE.DiagnosticLevel = Level;
 
   DE.WarningOption =
-      std::string(Info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
-          DE.DiagnosticID));
+      std::string(DiagnosticIDs::getWarningOptionForDiag(DE.DiagnosticID));
 
   // Format the message.
   SmallString<100> MessageStr;
@@ -161,3 +160,4 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
   // Record the diagnostic entry.
   Entries.push_back(DE);
 }
+
diff --git a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
index d1db31763e500..0887b5a504f05 100644
--- a/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/SerializedDiagnosticPrinter.cpp
@@ -202,7 +202,7 @@ class SDiagsWriter : public DiagnosticConsumer {
 
   /// Emit the string information for diagnostic flags.
   unsigned getEmitDiagnosticFlag(DiagnosticsEngine::Level DiagLevel,
-                                 const Diagnostic *Diag = nullptr);
+                                 unsigned DiagID = 0);
 
   unsigned getEmitDiagnosticFlag(StringRef DiagName);
 
@@ -536,13 +536,11 @@ unsigned SDiagsWriter::getEmitCategory(unsigned int category) {
 }
 
 unsigned SDiagsWriter::getEmitDiagnosticFlag(DiagnosticsEngine::Level DiagLevel,
-                                             const Diagnostic *Diag) {
-  if (!Diag || DiagLevel == DiagnosticsEngine::Note)
+                                             unsigned DiagID) {
+  if (DiagLevel == DiagnosticsEngine::Note)
     return 0; // No flag for notes.
 
-  StringRef FlagName =
-      Diag->getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
-          Diag->getID());
+  StringRef FlagName = DiagnosticIDs::getWarningOptionForDiag(DiagID);
   return getEmitDiagnosticFlag(FlagName);
 }
 
@@ -657,7 +655,7 @@ void SDiagsWriter::EmitDiagnosticMessage(FullSourceLoc Loc, PresumedLoc PLoc,
     unsigned DiagID = DiagnosticIDs::getCategoryNumberForDiag(Info->getID());
     Record.push_back(getEmitCategory(DiagID));
     // Emit the diagnostic flag string lazily and get the mapped ID.
-    Record.push_back(getEmitDiagnosticFlag(Level, Info));
+    Record.push_back(getEmitDiagnosticFlag(Level, Info->getID()));
   } else {
     Record.push_back(getEmitCategory());
     Record.push_back(getEmitDiagnosticFlag(Level));
diff --git a/clang/lib/Frontend/TextDiagnosticPrinter.cpp b/clang/lib/Frontend/TextDiagnosticPrinter.cpp
index 28f7218dc23f5..dac5c44fe9256 100644
--- a/clang/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/clang/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -70,17 +70,13 @@ static void printDiagnosticOptions(raw_ostream &OS,
     // flag it as such. Note that diagnostics could also have been mapped by a
     // pragma, but we don't currently have a way to distinguish this.
     if (Level == DiagnosticsEngine::Error &&
-        Info.getDiags()->getDiagnosticIDs()->isWarningOrExtension(
-            Info.getID()) &&
-        !Info.getDiags()->getDiagnosticIDs()->isDefaultMappingAsError(
-            Info.getID())) {
+        DiagnosticIDs::isBuiltinWarningOrExtension(Info.getID()) &&
+        !DiagnosticIDs::isDefaultMappingAsError(Info.getID())) {
       OS << " [-Werror";
       Started = true;
     }
 
-    StringRef Opt =
-        Info.getDiags()->getDiagnosticIDs()->getWarningOptionForDiag(
-            Info.getID());
+    StringRef Opt = DiagnosticIDs::getWarningOptionForDiag(Info.getID());
     if (!Opt.empty()) {
       OS << (Started ? "," : " [")
          << (Level == DiagnosticsEngine::Remark ? "-R" : "-W") << Opt;
diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp
index d19f79b6ddefc..ca521dc0bcd26 100644
--- a/clang/lib/Sema/HLSLExternalSemaSource.cpp
+++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp
@@ -193,36 +193,8 @@ struct BuiltinTypeDeclBuilder {
         ExplicitSpecifier(), false, true, false,
         ConstexprSpecKind::Unspecified);
 
-    DeclRefExpr *Fn =
-        lookupBuiltinFunction(AST, S, "__builtin_hlsl_create_handle");
-    Expr *RCExpr = emitResourceClassExpr(AST, RC);
-    Expr *Call = CallExpr::Create(AST, Fn, {RCExpr}, AST.VoidPtrTy, VK_PRValue,
-                                  SourceLocation(), FPOptionsOverride());
-
-    CXXThisExpr *This = CXXThisExpr::Create(
-        AST, SourceLocation(), Constructor->getFunctionObjectParameterType(),
-        true);
-    Expr *Handle = MemberExpr::CreateImplicit(AST, This, false, Fields["h"],
-                                              Fields["h"]->getType(), VK_LValue,
-                                              OK_Ordinary);
-
-    // If the handle isn't a void pointer, cast the builtin result to the
-    // correct type.
-    if (Handle->getType().getCanonicalType() != AST.VoidPtrTy) {
-      Call = CXXStaticCastExpr::Create(
-          AST, Handle->getType(), VK_PRValue, CK_Dependent, Call, nullptr,
-          AST.getTrivialTypeSourceInfo(Handle->getType(), SourceLocation()),
-          FPOptionsOverride(), SourceLocation(), SourceLocation(),
-          SourceRange());
-    }
-
-    BinaryOperator *Assign = BinaryOperator::Create(
-        AST, Handle, Call, BO_Assign, Handle->getType(), VK_LValue, OK_Ordinary,
-        SourceLocation(), FPOptionsOverride());
-
-    Constructor->setBody(
-        CompoundStmt::Create(AST, {Assign}, FPOptionsOverride(),
-                             SourceLocation(), SourceLocation()));
+    Constructor->setBody(CompoundStmt::Create(
+        AST, {}, FPOptionsOverride(), SourceLocation(), SourceLocation()));
     Constructor->setAccess(AccessSpecifier::AS_public);
     Record->addDecl(Constructor);
     return *this;
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 6d7a57d7b5a41..03dd39bf03a20 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1676,7 +1676,7 @@ void Sema::EmitDiagnostic(unsigned DiagID, const DiagnosticBuilder &DB) {
   // that is different from the last template instantiation where
   // we emitted an error, print a template instantiation
   // backtrace.
-  if (!Diags.getDiagnosticIDs()->isNote(DiagID))
+  if (!DiagnosticIDs::isBuiltinNote(DiagID))
     PrintContextStack();
 }
 
@@ -1690,8 +1690,7 @@ bool Sema::hasUncompilableErrorOccurred() const {
   if (Loc == DeviceDeferredDiags.end())
     return false;
   for (auto PDAt : Loc->second) {
-    if (Diags.getDiagnosticIDs()->isDefaultMappingAsError(
-            PDAt.second.getDiagID()))
+    if (DiagnosticIDs::isDefaultMappingAsError(PDAt.second.getDiagID()))
       return true;
   }
   return false;
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index fbb3de4b3e416..ec37c0df56c67 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -835,7 +835,7 @@ SemaBase::SemaDiagnosticBuilder SemaCUDA::DiagIfDeviceCode(SourceLocation Loc,
       if (!getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
       if (SemaRef.IsLastErrorImmediate &&
-          getDiagnostics().getDiagnosticIDs()->isNote(DiagID))
+          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
       return (SemaRef.getEmissionStatus(CurFunContext) ==
               Sema::FunctionEmissionStatus::Emitted)
@@ -866,7 +866,7 @@ Sema::SemaDiagnosticBuilder SemaCUDA::DiagIfHostCode(SourceLocation Loc,
       if (getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
       if (SemaRef.IsLastErrorImmediate &&
-          getDiagnostics().getDiagnosticIDs()->isNote(DiagID))
+          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
       return (SemaRef.getEmissionStatus(CurFunContext) ==
               Sema::FunctionEmissionStatus::Emitted)
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 14cc51cf89665..c9b9f3a0007da 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -852,38 +852,22 @@ static void handleDiagnoseIfAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (!checkFunctionConditionAttr(S, D, AL, Cond, Msg))
     return;
 
-  StringRef DefaultSevStr;
-  if (!S.checkStringLiteralArgumentAttr(AL, 2, DefaultSevStr))
+  StringRef DiagTypeStr;
+  if (!S.checkStringLiteralArgumentAttr(AL, 2, DiagTypeStr))
     return;
 
-  DiagnoseIfAttr::DefaultSeverity DefaultSev;
-  if (!DiagnoseIfAttr::ConvertStrToDefaultSeverity(DefaultSevStr, DefaultSev)) {
+  DiagnoseIfAttr::DiagnosticType DiagType;
+  if (!DiagnoseIfAttr::ConvertStrToDiagnosticType(DiagTypeStr, DiagType)) {
     S.Diag(AL.getArgAsExpr(2)->getBeginLoc(),
            diag::err_diagnose_if_invalid_diagnostic_type);
     return;
   }
 
-  StringRef WarningGroup;
-  SmallVector<StringRef, 2> Options;
-  if (AL.getNumArgs() > 3) {
-    if (!S.checkStringLiteralArgumentAttr(AL, 3, WarningGroup))
-      return;
-    if (WarningGroup.empty() ||
-        !S.getDiagnostics().getDiagnosticIDs()->getGroupForWarningOption(
-            WarningGroup)) {
-      S.Diag(AL.getArgAsExpr(3)->getBeginLoc(),
-             diag::err_diagnose_if_unknown_warning)
-          << WarningGroup;
-      return;
-    }
-  }
-
   bool ArgDependent = false;
   if (const auto *FD = dyn_cast<FunctionDecl>(D))
     ArgDependent = ArgumentDependenceChecker(FD).referencesArgs(Cond);
   D->addAttr(::new (S.Context) DiagnoseIfAttr(
-      S.Context, AL, Cond, Msg, DefaultSev, WarningGroup, ArgDependent,
-      cast<NamedDecl>(D)));
+      S.Context, AL, Cond, Msg, DiagType, ArgDependent, cast<NamedDecl>(D)));
 }
 
 static void handleNoBuiltinAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index ebe76185cbb2d..1d8ccdda45573 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -693,13 +693,19 @@ bool clang::CreateHLSLAttributedResourceType(
 // HLSL resource. The attributes are collected in HLSLResourcesTypeAttrs and at
 // the end of the declaration they are applied to the declaration type by
 // wrapping it in HLSLAttributedResourceType.
-bool SemaHLSL::handleResourceTypeAttr(const ParsedAttr &AL) {
-  Attr *A = nullptr;
+bool SemaHLSL::handleResourceTypeAttr(QualType T, const ParsedAttr &AL) {
+  // only allow resource type attributes on intangible types
+  if (!T->isHLSLResourceType()) {
+    Diag(AL.getLoc(), diag::err_hlsl_attribute_needs_intangible_type)
+        << AL << getASTContext().HLSLResourceTy;
+    return false;
+  }
 
   // validate number of arguments
   if (!AL.checkExactlyNumArgs(SemaRef, AL.getMinArgs()))
     return false;
 
+  Attr *A = nullptr;
   switch (AL.getKind()) {
   case ParsedAttr::AT_HLSLResourceClass: {
     if (!AL.isArgIdent(0)) {
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index d304f322aced6..0c1e054f7c30a 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -7300,10 +7300,8 @@ static bool diagnoseDiagnoseIfAttrsWith(Sema &S, const NamedDecl *ND,
     return false;
 
   auto WarningBegin = std::stable_partition(
-      Attrs.begin(), Attrs.end(), [](const DiagnoseIfAttr *DIA) {
-        return DIA->getDefaultSeverity() == DiagnoseIfAttr::DS_error &&
-               DIA->getWarningGroup().empty();
-      });
+      Attrs.begin(), Attrs.end(),
+      [](const DiagnoseIfAttr *DIA) { return DIA->isError(); });
 
   // Note that diagnose_if attributes are late-parsed, so they appear in the
   // correct order (unlike enable_if attributes).
@@ -7317,32 +7315,11 @@ static bool diagnoseDiagnoseIfAttrsWith(Sema &S, const NamedDecl *ND,
     return true;
   }
 
-  auto ToSeverity = [](DiagnoseIfAttr::DefaultSeverity Sev) {
-    switch (Sev) {
-    case DiagnoseIfAttr::DS_warning:
-      return diag::Severity::Warning;
-    case DiagnoseIfAttr::DS_error:
-      return diag::Severity::Error;
-    }
-    llvm_unreachable("Fully covered switch above!");
-  };
-
   for (const auto *DIA : llvm::make_range(WarningBegin, Attrs.end()))
     if (IsSuccessful(DIA)) {
-      if (DIA->getWarningGroup().empty() &&
-          DIA->getDefaultSeverity() == DiagnoseIfAttr::DS_warning) {
-        S.Diag(Loc, diag::warn_diagnose_if_succeeded) << DIA->getMessage();
-        S.Diag(DIA->getLocation(), diag::note_from_diagnose_if)
-            << DIA->getParent() << DIA->getCond()->getSourceRange();
-      } else {
-        auto DiagGroup = S.Diags.getDiagnosticIDs()->getGroupForWarningOption(
-            DIA->getWarningGroup());
-        assert(DiagGroup);
-        auto DiagID = S.Diags.getDiagnosticIDs()->getCustomDiagID(
-            {ToSeverity(DIA->getDefaultSeverity()), "%0",
-             DiagnosticIDs::CLASS_WARNING, false, false, *DiagGroup});
-        S.Diag(Loc, DiagID) << DIA->getMessage();
-      }
+      S.Diag(Loc, diag::warn_diagnose_if_succeeded) << DIA->getMessage();
+      S.Diag(DIA->getLocation(), diag::note_from_diagnose_if)
+          << DIA->getParent() << DIA->getCond()->getSourceRange();
     }
 
   return false;
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 55f38743e2768..fd51fa4afcacb 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -1677,7 +1677,8 @@ namespace {
       // Lambdas have already been processed inside their eval contexts.
       if (SemaRef.RebuildingImmediateInvocation)
         return E;
-      LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
+      LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true,
+                                    /*InstantiatingLambda=*/true);
       Sema::ConstraintEvalRAII<TemplateInstantiator> RAII(*this);
 
       return inherited::TransformLambdaExpr(E);
@@ -2432,8 +2433,18 @@ QualType TemplateInstantiator::TransformFunctionProtoType(TypeLocBuilder &TLB,
                                  CXXRecordDecl *ThisContext,
                                  Qualifiers ThisTypeQuals,
                                  Fn TransformExceptionSpec) {
-  // We need a local instantiation scope for this function prototype.
-  LocalInstantiationScope Scope(SemaRef, /*CombineWithOuterScope=*/true);
+  // If this is a lambda, the transformation MUST be done in the
+  // CurrentInstantiationScope since it introduces a mapping of
+  // the original to the newly created transformed parameters.
+  //
+  // In that case, TemplateInstantiator::TransformLambdaExpr will
+  // have already pushed a scope for this prototype, so don't create
+  // a second one.
+  LocalInstantiationScope *Current = getSema().CurrentInstantiationScope;
+  std::optional<LocalInstantiationScope> Scope;
+  if (!Current || !Current->isLambda())
+    Scope.emplace(SemaRef, /*CombineWithOuterScope=*/true);
+
   return inherited::TransformFunctionProtoType(
       TLB, TL, ThisContext, ThisTypeQuals, TransformExceptionSpec);
 }
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index e055c87e78381..c3cb9d5d8c2c3 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -284,8 +284,7 @@ static void instantiateDependentDiagnoseIfAttr(
   if (Cond)
     New->addAttr(new (S.getASTContext()) DiagnoseIfAttr(
         S.getASTContext(), *DIA, Cond, DIA->getMessage(),
-        DIA->getDefaultSeverity(), DIA->getWarningGroup(),
-        DIA->getArgDependent(), New));
+        DIA->getDiagnosticType(), DIA->getArgDependent(), New));
 }
 
 // Constructs and adds to New a new instance of CUDALaunchBoundsAttr using
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 950bd6db0359d..a7beb9d222c3b 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -8860,7 +8860,7 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type,
       // decl-specifier-seq; do not collect attributes on declarations or those
       // that get to slide after declaration name.
       if (TAL == TAL_DeclSpec &&
-          state.getSema().HLSL().handleResourceTypeAttr(attr))
+          state.getSema().HLSL().handleResourceTypeAttr(type, attr))
         attr.setUsedAsTypeAttr();
       break;
     }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 95ded5e59a9fa..91cb980ee26b2 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -684,10 +684,6 @@ class TreeTransform {
                                       Qualifiers ThisTypeQuals,
                                       Fn TransformExceptionSpec);
 
-  template <typename Fn>
-  QualType TransformAttributedType(TypeLocBuilder &TLB, AttributedTypeLoc TL,
-                                   Fn TransformModifiedType);
-
   bool TransformExceptionSpec(SourceLocation Loc,
                               FunctionProtoType::ExceptionSpecInfo &ESI,
                               SmallVectorImpl<QualType> &Exceptions,
@@ -7373,11 +7369,10 @@ TreeTransform<Derived>::TransformElaboratedType(TypeLocBuilder &TLB,
 }
 
 template <typename Derived>
-template <typename Fn>
-QualType TreeTransform<Derived>::TransformAttributedType(
-    TypeLocBuilder &TLB, AttributedTypeLoc TL, Fn TransformModifiedTypeFn) {
+QualType TreeTransform<Derived>::TransformAttributedType(TypeLocBuilder &TLB,
+                                                         AttributedTypeLoc TL) {
   const AttributedType *oldType = TL.getTypePtr();
-  QualType modifiedType = TransformModifiedTypeFn(TLB, TL.getModifiedLoc());
+  QualType modifiedType = getDerived().TransformType(TLB, TL.getModifiedLoc());
   if (modifiedType.isNull())
     return QualType();
 
@@ -7392,12 +7387,27 @@ QualType TreeTransform<Derived>::TransformAttributedType(
   // FIXME: dependent operand expressions?
   if (getDerived().AlwaysRebuild() ||
       modifiedType != oldType->getModifiedType()) {
-    TypeLocBuilder AuxiliaryTLB;
-    AuxiliaryTLB.reserve(TL.getFullDataSize());
-    QualType equivalentType =
-        getDerived().TransformType(AuxiliaryTLB, TL.getEquivalentTypeLoc());
-    if (equivalentType.isNull())
-      return QualType();
+    // If the equivalent type is equal to the modified type, we don't want to
+    // transform it as well because:
+    //
+    //   1. The transformation would yield the same result and is therefore
+    //      superfluous, and
+    //
+    //   2. Transforming the same type twice can cause problems, e.g. if it
+    //      is a FunctionProtoType, we may end up instantiating the function
+    //      parameters twice, which causes an assertion since the parameters
+    //      are already bound to their counterparts in the template for this
+    //      instantiation.
+    //
+    QualType equivalentType = modifiedType;
+    if (TL.getModifiedLoc().getType() != TL.getEquivalentTypeLoc().getType()) {
+      TypeLocBuilder AuxiliaryTLB;
+      AuxiliaryTLB.reserve(TL.getFullDataSize());
+      equivalentType =
+          getDerived().TransformType(AuxiliaryTLB, TL.getEquivalentTypeLoc());
+      if (equivalentType.isNull())
+        return QualType();
+    }
 
     // Check whether we can add nullability; it is only represented as
     // type sugar, and therefore cannot be diagnosed in any other way.
@@ -7421,15 +7431,6 @@ QualType TreeTransform<Derived>::TransformAttributedType(
   return result;
 }
 
-template <typename Derived>
-QualType TreeTransform<Derived>::TransformAttributedType(TypeLocBuilder &TLB,
-                                                         AttributedTypeLoc TL) {
-  return getDerived().TransformAttributedType(
-      TLB, TL, [&](TypeLocBuilder &TLB, TypeLoc ModifiedLoc) -> QualType {
-        return getDerived().TransformType(TLB, ModifiedLoc);
-      });
-}
-
 template <typename Derived>
 QualType TreeTransform<Derived>::TransformCountAttributedType(
     TypeLocBuilder &TLB, CountAttributedTypeLoc TL) {
@@ -14774,63 +14775,29 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
         TPL->containsUnexpandedParameterPack();
   }
 
-  // Transform the type of the original lambda's call operator.
-  // The transformation MUST be done in the CurrentInstantiationScope since
-  // it introduces a mapping of the original to the newly created
-  // transformed parameters.
-  TypeSourceInfo *NewCallOpTSI = nullptr;
-  {
-    auto OldCallOpTypeLoc =
-        E->getCallOperator()->getTypeSourceInfo()->getTypeLoc();
-
-    auto TransformFunctionProtoTypeLoc =
-        [this](TypeLocBuilder &TLB, FunctionProtoTypeLoc FPTL) -> QualType {
-      SmallVector<QualType, 4> ExceptionStorage;
-      return this->TransformFunctionProtoType(
-          TLB, FPTL, nullptr, Qualifiers(),
-          [&](FunctionProtoType::ExceptionSpecInfo &ESI, bool &Changed) {
-            return TransformExceptionSpec(FPTL.getBeginLoc(), ESI,
-                                          ExceptionStorage, Changed);
-          });
-    };
-
-    QualType NewCallOpType;
-    TypeLocBuilder NewCallOpTLBuilder;
-
-    if (auto ATL = OldCallOpTypeLoc.getAs<AttributedTypeLoc>()) {
-      NewCallOpType = this->TransformAttributedType(
-          NewCallOpTLBuilder, ATL,
-          [&](TypeLocBuilder &TLB, TypeLoc TL) -> QualType {
-            return TransformFunctionProtoTypeLoc(
-                TLB, TL.castAs<FunctionProtoTypeLoc>());
-          });
-    } else {
-      auto FPTL = OldCallOpTypeLoc.castAs<FunctionProtoTypeLoc>();
-      NewCallOpType = TransformFunctionProtoTypeLoc(NewCallOpTLBuilder, FPTL);
-    }
-
-    if (NewCallOpType.isNull())
-      return ExprError();
-    LSI->ContainsUnexpandedParameterPack |=
-        NewCallOpType->containsUnexpandedParameterPack();
-    NewCallOpTSI =
-        NewCallOpTLBuilder.getTypeSourceInfo(getSema().Context, NewCallOpType);
-  }
+  TypeLocBuilder NewCallOpTLBuilder;
+  TypeLoc OldCallOpTypeLoc =
+      E->getCallOperator()->getTypeSourceInfo()->getTypeLoc();
+  QualType NewCallOpType =
+      getDerived().TransformType(NewCallOpTLBuilder, OldCallOpTypeLoc);
+  if (NewCallOpType.isNull())
+    return ExprError();
+  LSI->ContainsUnexpandedParameterPack |=
+      NewCallOpType->containsUnexpandedParameterPack();
+  TypeSourceInfo *NewCallOpTSI =
+      NewCallOpTLBuilder.getTypeSourceInfo(getSema().Context, NewCallOpType);
 
-  ArrayRef<ParmVarDecl *> Params;
-  if (auto ATL = NewCallOpTSI->getTypeLoc().getAs<AttributedTypeLoc>()) {
-    Params = ATL.getModifiedLoc().castAs<FunctionProtoTypeLoc>().getParams();
-  } else {
-    auto FPTL = NewCallOpTSI->getTypeLoc().castAs<FunctionProtoTypeLoc>();
-    Params = FPTL.getParams();
-  }
+  // The type may be an AttributedType or some other kind of sugar;
+  // get the actual underlying FunctionProtoType.
+  auto FPTL = NewCallOpTSI->getTypeLoc().getAsAdjusted<FunctionProtoTypeLoc>();
+  assert(FPTL && "Not a FunctionProtoType?");
 
   getSema().CompleteLambdaCallOperator(
       NewCallOperator, E->getCallOperator()->getLocation(),
       E->getCallOperator()->getInnerLocStart(),
       E->getCallOperator()->getTrailingRequiresClause(), NewCallOpTSI,
       E->getCallOperator()->getConstexprKind(),
-      E->getCallOperator()->getStorageClass(), Params,
+      E->getCallOperator()->getStorageClass(), FPTL.getParams(),
       E->hasExplicitResultType());
 
   getDerived().transformAttrs(E->getCallOperator(), NewCallOperator);
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 1f7946e61d175..8623c030b6d59 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -6647,7 +6647,7 @@ void ASTReader::ReadPragmaDiagnosticMappings(DiagnosticsEngine &Diag) {
       // command line (-w, -Weverything, -Werror, ...) along with any explicit
       // -Wblah flags.
       unsigned Flags = Record[Idx++];
-      DiagState Initial(*Diag.getDiagnosticIDs());
+      DiagState Initial;
       Initial.SuppressSystemWarnings = Flags & 1; Flags >>= 1;
       Initial.ErrorsAsFatal = Flags & 1; Flags >>= 1;
       Initial.WarningsAsErrors = Flags & 1; Flags >>= 1;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index f326e3c2e2ff7..7a40c5c65d39d 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3220,7 +3220,7 @@ void ASTWriter::WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag,
         // Skip default mappings. We have a mapping for every diagnostic ever
         // emitted, regardless of whether it was customized.
         if (!I.second.isPragma() &&
-            I.second == Diag.getDiagnosticIDs()->getDefaultMapping(I.first))
+            I.second == DiagnosticIDs::getDefaultMapping(I.first))
           continue;
         Mappings.push_back(I);
       }
@@ -5713,8 +5713,7 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
     // efficent becuase it allows lazy deserialization.
     RecordData FunctionToLambdasMapRecord;
     for (const auto &Pair : FunctionToLambdasMap) {
-      FunctionToLambdasMapRecord.push_back(
-          GetDeclRef(Pair.first).getRawValue());
+      FunctionToLambdasMapRecord.push_back(Pair.first.getRawValue());
       FunctionToLambdasMapRecord.push_back(Pair.second.size());
       for (const auto &Lambda : Pair.second)
         FunctionToLambdasMapRecord.push_back(Lambda.getRawValue());
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 50c090b195d61..b9222a1b33fd7 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1524,7 +1524,8 @@ void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
     // For lambdas inside canonical FunctionDecl remember the mapping.
     if (auto FD = llvm::dyn_cast_or_null<FunctionDecl>(D->getDeclContext());
         FD && FD->isCanonicalDecl()) {
-      Writer.FunctionToLambdasMap[FD].push_back(Writer.GetDeclRef(D));
+      Writer.FunctionToLambdasMap[Writer.GetDeclRef(FD)].push_back(
+          Writer.GetDeclRef(D));
     }
   } else {
     Record.push_back(CXXRecNotTemplate);
diff --git a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 414282d58f779..6da3665ab9a4d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -132,7 +132,7 @@ add_clang_library(clangStaticAnalyzerCheckers
   VLASizeChecker.cpp
   ValistChecker.cpp
   VirtualCallChecker.cpp
-  WebKit/NoUncountedMembersChecker.cpp
+  WebKit/RawPtrRefMemberChecker.cpp
   WebKit/ASTUtils.cpp
   WebKit/PtrTypesSemantics.cpp
   WebKit/RefCntblBaseVirtualDtorChecker.cpp
diff --git a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index 7aefcdc6d358a..e7fde3edc7f9e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -48,6 +48,9 @@ void FixedAddressChecker::checkPreStmt(const BinaryOperator *B,
   if (!RV.isConstant() || RV.isZeroConstant())
     return;
 
+  if (C.getSourceManager().isInSystemMacro(B->getRHS()->getBeginLoc()))
+    return;
+
   if (ExplodedNode *N = C.generateNonFatalErrorNode()) {
     // FIXME: improve grammar in the following strings:
     constexpr llvm::StringLiteral Msg =
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 81ec8e1b51698..3e95db7e97fac 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -1811,9 +1811,9 @@ MallocChecker::MallocMemReturnsAttr(CheckerContext &C, const CallEvent &Call,
   if (!Att->args().empty()) {
     return MallocMemAux(C, Call,
                         Call.getArgExpr(Att->args_begin()->getASTIndex()),
-                        UndefinedVal(), State, Family);
+                        UnknownVal(), State, Family);
   }
-  return MallocMemAux(C, Call, UnknownVal(), UndefinedVal(), State, Family);
+  return MallocMemAux(C, Call, UnknownVal(), UnknownVal(), State, Family);
 }
 
 ProgramStateRef MallocChecker::MallocBindRetVal(CheckerContext &C,
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 9da3e54e45431..4d145be808f6d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -19,8 +19,7 @@ using namespace clang;
 
 namespace {
 
-bool hasPublicMethodInBaseClass(const CXXRecordDecl *R,
-                                const char *NameToMatch) {
+bool hasPublicMethodInBaseClass(const CXXRecordDecl *R, StringRef NameToMatch) {
   assert(R);
   assert(R->hasDefinition());
 
@@ -37,7 +36,7 @@ bool hasPublicMethodInBaseClass(const CXXRecordDecl *R,
 namespace clang {
 
 std::optional<const clang::CXXRecordDecl *>
-hasPublicMethodInBase(const CXXBaseSpecifier *Base, const char *NameToMatch) {
+hasPublicMethodInBase(const CXXBaseSpecifier *Base, StringRef NameToMatch) {
   assert(Base);
 
   const Type *T = Base->getType().getTypePtrOrNull();
@@ -53,16 +52,17 @@ hasPublicMethodInBase(const CXXBaseSpecifier *Base, const char *NameToMatch) {
   return hasPublicMethodInBaseClass(R, NameToMatch) ? R : nullptr;
 }
 
-std::optional<bool> isRefCountable(const CXXRecordDecl* R)
-{
+std::optional<bool> isSmartPtrCompatible(const CXXRecordDecl *R,
+                                         StringRef IncMethodName,
+                                         StringRef DecMethodName) {
   assert(R);
 
   R = R->getDefinition();
   if (!R)
     return std::nullopt;
 
-  bool hasRef = hasPublicMethodInBaseClass(R, "ref");
-  bool hasDeref = hasPublicMethodInBaseClass(R, "deref");
+  bool hasRef = hasPublicMethodInBaseClass(R, IncMethodName);
+  bool hasDeref = hasPublicMethodInBaseClass(R, DecMethodName);
   if (hasRef && hasDeref)
     return true;
 
@@ -70,15 +70,15 @@ std::optional<bool> isRefCountable(const CXXRecordDecl* R)
   Paths.setOrigin(const_cast<CXXRecordDecl *>(R));
 
   bool AnyInconclusiveBase = false;
-  const auto hasPublicRefInBase =
-      [&AnyInconclusiveBase](const CXXBaseSpecifier *Base, CXXBasePath &) {
-        auto hasRefInBase = clang::hasPublicMethodInBase(Base, "ref");
-        if (!hasRefInBase) {
-          AnyInconclusiveBase = true;
-          return false;
-        }
-        return (*hasRefInBase) != nullptr;
-      };
+  const auto hasPublicRefInBase = [&](const CXXBaseSpecifier *Base,
+                                      CXXBasePath &) {
+    auto hasRefInBase = clang::hasPublicMethodInBase(Base, IncMethodName);
+    if (!hasRefInBase) {
+      AnyInconclusiveBase = true;
+      return false;
+    }
+    return (*hasRefInBase) != nullptr;
+  };
 
   hasRef = hasRef || R->lookupInBases(hasPublicRefInBase, Paths,
                                       /*LookupInDependent =*/true);
@@ -86,15 +86,15 @@ std::optional<bool> isRefCountable(const CXXRecordDecl* R)
     return std::nullopt;
 
   Paths.clear();
-  const auto hasPublicDerefInBase =
-      [&AnyInconclusiveBase](const CXXBaseSpecifier *Base, CXXBasePath &) {
-        auto hasDerefInBase = clang::hasPublicMethodInBase(Base, "deref");
-        if (!hasDerefInBase) {
-          AnyInconclusiveBase = true;
-          return false;
-        }
-        return (*hasDerefInBase) != nullptr;
-      };
+  const auto hasPublicDerefInBase = [&](const CXXBaseSpecifier *Base,
+                                        CXXBasePath &) {
+    auto hasDerefInBase = clang::hasPublicMethodInBase(Base, DecMethodName);
+    if (!hasDerefInBase) {
+      AnyInconclusiveBase = true;
+      return false;
+    }
+    return (*hasDerefInBase) != nullptr;
+  };
   hasDeref = hasDeref || R->lookupInBases(hasPublicDerefInBase, Paths,
                                           /*LookupInDependent =*/true);
   if (AnyInconclusiveBase)
@@ -103,11 +103,23 @@ std::optional<bool> isRefCountable(const CXXRecordDecl* R)
   return hasRef && hasDeref;
 }
 
+std::optional<bool> isRefCountable(const clang::CXXRecordDecl *R) {
+  return isSmartPtrCompatible(R, "ref", "deref");
+}
+
+std::optional<bool> isCheckedPtrCapable(const clang::CXXRecordDecl *R) {
+  return isSmartPtrCompatible(R, "incrementPtrCount", "decrementPtrCount");
+}
+
 bool isRefType(const std::string &Name) {
   return Name == "Ref" || Name == "RefAllowingPartiallyDestroyed" ||
          Name == "RefPtr" || Name == "RefPtrAllowingPartiallyDestroyed";
 }
 
+bool isCheckedPtr(const std::string &Name) {
+  return Name == "CheckedPtr" || Name == "CheckedRef";
+}
+
 bool isCtorOfRefCounted(const clang::FunctionDecl *F) {
   assert(F);
   const std::string &FunctionName = safeGetName(F);
@@ -155,7 +167,7 @@ std::optional<bool> isUncounted(const QualType T) {
 std::optional<bool> isUncounted(const CXXRecordDecl* Class)
 {
   // Keep isRefCounted first as it's cheaper.
-  if (isRefCounted(Class))
+  if (!Class || isRefCounted(Class))
     return false;
 
   std::optional<bool> IsRefCountable = isRefCountable(Class);
@@ -217,6 +229,15 @@ bool isRefCounted(const CXXRecordDecl *R) {
   return false;
 }
 
+bool isCheckedPtr(const CXXRecordDecl *R) {
+  assert(R);
+  if (auto *TmplR = R->getTemplateInstantiationPattern()) {
+    const auto &ClassName = safeGetName(TmplR);
+    return isCheckedPtr(ClassName);
+  }
+  return false;
+}
+
 bool isPtrConversion(const FunctionDecl *F) {
   assert(F);
   if (isCtorOfRefCounted(F))
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
index e2d0342bebd52..3528c52a7d659 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.h
@@ -34,15 +34,23 @@ class Type;
 /// \returns CXXRecordDecl of the base if the type has ref as a public method,
 /// nullptr if not, std::nullopt if inconclusive.
 std::optional<const clang::CXXRecordDecl *>
-hasPublicMethodInBase(const CXXBaseSpecifier *Base, const char *NameToMatch);
+hasPublicMethodInBase(const CXXBaseSpecifier *Base,
+                      llvm::StringRef NameToMatch);
 
 /// \returns true if \p Class is ref-countable, false if not, std::nullopt if
 /// inconclusive.
-std::optional<bool> isRefCountable(const clang::CXXRecordDecl* Class);
+std::optional<bool> isRefCountable(const clang::CXXRecordDecl *Class);
+
+/// \returns true if \p Class is checked-pointer compatible, false if not,
+/// std::nullopt if inconclusive.
+std::optional<bool> isCheckedPtrCapable(const clang::CXXRecordDecl *Class);
 
 /// \returns true if \p Class is ref-counted, false if not.
 bool isRefCounted(const clang::CXXRecordDecl *Class);
 
+/// \returns true if \p Class is a CheckedPtr / CheckedRef, false if not.
+bool isCheckedPtr(const clang::CXXRecordDecl *Class);
+
 /// \returns true if \p Class is ref-countable AND not ref-counted, false if
 /// not, std::nullopt if inconclusive.
 std::optional<bool> isUncounted(const clang::QualType T);
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
similarity index 63%
rename from clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp
rename to clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
index 69a0eb3086ab7..2ce6bc330e0ca 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/NoUncountedMembersChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RawPtrRefMemberChecker.cpp
@@ -1,4 +1,4 @@
-//=======- NoUncountedMembersChecker.cpp -------------------------*- C++ -*-==//
+//=======- RawPtrRefMemberChecker.cpp ----------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -25,18 +25,21 @@ using namespace ento;
 
 namespace {
 
-class NoUncountedMemberChecker
+class RawPtrRefMemberChecker
     : public Checker<check::ASTDecl<TranslationUnitDecl>> {
 private:
   BugType Bug;
   mutable BugReporter *BR;
 
 public:
-  NoUncountedMemberChecker()
-      : Bug(this,
-            "Member variable is a raw-pointer/reference to reference-countable "
-            "type",
-            "WebKit coding guidelines") {}
+  RawPtrRefMemberChecker(const char *description)
+      : Bug(this, description, "WebKit coding guidelines") {}
+
+  virtual std::optional<bool>
+  isPtrCompatible(const clang::CXXRecordDecl *) const = 0;
+  virtual bool isPtrCls(const clang::CXXRecordDecl *) const = 0;
+  virtual const char *typeName() const = 0;
+  virtual const char *invariant() const = 0;
 
   void checkASTDecl(const TranslationUnitDecl *TUD, AnalysisManager &MGR,
                     BugReporter &BRArg) const {
@@ -46,8 +49,8 @@ class NoUncountedMemberChecker
     // visit template instantiations or lambda classes. We
     // want to visit those, so we make our own RecursiveASTVisitor.
     struct LocalVisitor : public RecursiveASTVisitor<LocalVisitor> {
-      const NoUncountedMemberChecker *Checker;
-      explicit LocalVisitor(const NoUncountedMemberChecker *Checker)
+      const RawPtrRefMemberChecker *Checker;
+      explicit LocalVisitor(const RawPtrRefMemberChecker *Checker)
           : Checker(Checker) {
         assert(Checker);
       }
@@ -77,9 +80,9 @@ class NoUncountedMemberChecker
       if (auto *MemberCXXRD = MemberType->getPointeeCXXRecordDecl()) {
         // If we don't see the definition we just don't know.
         if (MemberCXXRD->hasDefinition()) {
-            std::optional<bool> isRCAble = isRefCountable(MemberCXXRD);
-            if (isRCAble && *isRCAble)
-                reportBug(Member, MemberType, MemberCXXRD, RD);
+          std::optional<bool> isRCAble = isPtrCompatible(MemberCXXRD);
+          if (isRCAble && *isRCAble)
+            reportBug(Member, MemberType, MemberCXXRD, RD);
         }
       }
     }
@@ -114,7 +117,7 @@ class NoUncountedMemberChecker
     // a member but we trust them to handle it correctly.
     auto CXXRD = llvm::dyn_cast_or_null<CXXRecordDecl>(RD);
     if (CXXRD)
-      return isRefCounted(CXXRD);
+      return isPtrCls(CXXRD);
 
     return false;
   }
@@ -134,10 +137,10 @@ class NoUncountedMemberChecker
     Os << " in ";
     printQuotedQualifiedName(Os, ClassCXXRD);
     Os << " is a "
-       << (isa<PointerType>(MemberType) ? "raw pointer" : "reference")
-       << " to ref-countable type ";
+       << (isa<PointerType>(MemberType) ? "raw pointer" : "reference") << " to "
+       << typeName() << " ";
     printQuotedQualifiedName(Os, MemberCXXRD);
-    Os << "; member variables must be ref-counted.";
+    Os << "; " << invariant() << ".";
 
     PathDiagnosticLocation BSLoc(Member->getSourceRange().getBegin(),
                                  BR->getSourceManager());
@@ -146,13 +149,67 @@ class NoUncountedMemberChecker
     BR->emitReport(std::move(Report));
   }
 };
+
+class NoUncountedMemberChecker final : public RawPtrRefMemberChecker {
+public:
+  NoUncountedMemberChecker()
+      : RawPtrRefMemberChecker("Member variable is a raw-pointer/reference to "
+                               "reference-countable type") {}
+
+  std::optional<bool>
+  isPtrCompatible(const clang::CXXRecordDecl *R) const final {
+    return isRefCountable(R);
+  }
+
+  bool isPtrCls(const clang::CXXRecordDecl *R) const final {
+    return isRefCounted(R);
+  }
+
+  const char *typeName() const final { return "ref-countable type"; }
+
+  const char *invariant() const final {
+    return "member variables must be Ref, RefPtr, WeakRef, or WeakPtr";
+  }
+};
+
+class NoUncheckedPtrMemberChecker final : public RawPtrRefMemberChecker {
+public:
+  NoUncheckedPtrMemberChecker()
+      : RawPtrRefMemberChecker("Member variable is a raw-pointer/reference to "
+                               "checked-pointer capable type") {}
+
+  std::optional<bool>
+  isPtrCompatible(const clang::CXXRecordDecl *R) const final {
+    return isCheckedPtrCapable(R);
+  }
+
+  bool isPtrCls(const clang::CXXRecordDecl *R) const final {
+    return isCheckedPtr(R);
+  }
+
+  const char *typeName() const final { return "CheckedPtr capable type"; }
+
+  const char *invariant() const final {
+    return "member variables must be a CheckedPtr, CheckedRef, WeakRef, or "
+           "WeakPtr";
+  }
+};
+
 } // namespace
 
 void ento::registerNoUncountedMemberChecker(CheckerManager &Mgr) {
   Mgr.registerChecker<NoUncountedMemberChecker>();
 }
 
-bool ento::shouldRegisterNoUncountedMemberChecker(
+bool ento::shouldRegisterNoUncountedMemberChecker(const CheckerManager &Mgr) {
+  return true;
+}
+
+void ento::registerNoUncheckedPtrMemberChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<NoUncheckedPtrMemberChecker>();
+}
+
+bool ento::shouldRegisterNoUncheckedPtrMemberChecker(
     const CheckerManager &Mgr) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
index 81c2434ce6477..0ed93ab26bf5c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedCallArgsChecker.cpp
@@ -18,6 +18,8 @@
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/SaveAndRestore.h"
 #include <optional>
 
 using namespace clang;
@@ -44,7 +46,11 @@ class UncountedCallArgsChecker
     // visit template instantiations or lambda classes. We
     // want to visit those, so we make our own RecursiveASTVisitor.
     struct LocalVisitor : public RecursiveASTVisitor<LocalVisitor> {
+      using Base = RecursiveASTVisitor<LocalVisitor>;
+
       const UncountedCallArgsChecker *Checker;
+      Decl *DeclWithIssue{nullptr};
+
       explicit LocalVisitor(const UncountedCallArgsChecker *Checker)
           : Checker(Checker) {
         assert(Checker);
@@ -56,12 +62,18 @@ class UncountedCallArgsChecker
       bool TraverseClassTemplateDecl(ClassTemplateDecl *Decl) {
         if (isRefType(safeGetName(Decl)))
           return true;
-        return RecursiveASTVisitor<LocalVisitor>::TraverseClassTemplateDecl(
-            Decl);
+        return Base::TraverseClassTemplateDecl(Decl);
+      }
+
+      bool TraverseDecl(Decl *D) {
+        llvm::SaveAndRestore SavedDecl(DeclWithIssue);
+        if (D && (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)))
+          DeclWithIssue = D;
+        return Base::TraverseDecl(D);
       }
 
       bool VisitCallExpr(const CallExpr *CE) {
-        Checker->visitCallExpr(CE);
+        Checker->visitCallExpr(CE, DeclWithIssue);
         return true;
       }
     };
@@ -70,7 +82,7 @@ class UncountedCallArgsChecker
     visitor.TraverseDecl(const_cast<TranslationUnitDecl *>(TUD));
   }
 
-  void visitCallExpr(const CallExpr *CE) const {
+  void visitCallExpr(const CallExpr *CE, const Decl *D) const {
     if (shouldSkipCall(CE))
       return;
 
@@ -86,10 +98,10 @@ class UncountedCallArgsChecker
             return;
         }
         auto *E = MemberCallExpr->getImplicitObjectArgument();
-        QualType ArgType = MemberCallExpr->getObjectType();
+        QualType ArgType = MemberCallExpr->getObjectType().getCanonicalType();
         std::optional<bool> IsUncounted = isUncounted(ArgType);
         if (IsUncounted && *IsUncounted && !isPtrOriginSafe(E))
-          reportBugOnThis(E);
+          reportBugOnThis(E, D);
       }
 
       for (auto P = F->param_begin();
@@ -102,12 +114,13 @@ class UncountedCallArgsChecker
         // if ((*P)->hasAttr<SafeRefCntblRawPtrAttr>())
         //  continue;
 
-        const auto *ArgType = (*P)->getType().getTypePtrOrNull();
-        if (!ArgType)
+        QualType ArgType = (*P)->getType().getCanonicalType();
+        const auto *TypePtr = ArgType.getTypePtrOrNull();
+        if (!TypePtr)
           continue; // FIXME? Should we bail?
 
         // FIXME: more complex types (arrays, references to raw pointers, etc)
-        std::optional<bool> IsUncounted = isUncountedPtr(ArgType);
+        std::optional<bool> IsUncounted = isUncountedPtr(TypePtr);
         if (!IsUncounted || !(*IsUncounted))
           continue;
 
@@ -119,7 +132,7 @@ class UncountedCallArgsChecker
         if (isPtrOriginSafe(Arg))
           continue;
 
-        reportBug(Arg, *P);
+        reportBug(Arg, *P, D);
       }
     }
   }
@@ -240,7 +253,8 @@ class UncountedCallArgsChecker
             ClsName.ends_with("String"));
   }
 
-  void reportBug(const Expr *CallArg, const ParmVarDecl *Param) const {
+  void reportBug(const Expr *CallArg, const ParmVarDecl *Param,
+                 const Decl *DeclWithIssue) const {
     assert(CallArg);
 
     SmallString<100> Buf;
@@ -261,10 +275,11 @@ class UncountedCallArgsChecker
     PathDiagnosticLocation BSLoc(SrcLocToReport, BR->getSourceManager());
     auto Report = std::make_unique<BasicBugReport>(Bug, Os.str(), BSLoc);
     Report->addRange(CallArg->getSourceRange());
+    Report->setDeclWithIssue(DeclWithIssue);
     BR->emitReport(std::move(Report));
   }
 
-  void reportBugOnThis(const Expr *CallArg) const {
+  void reportBugOnThis(const Expr *CallArg, const Decl *DeclWithIssue) const {
     assert(CallArg);
 
     const SourceLocation SrcLocToReport = CallArg->getSourceRange().getBegin();
@@ -274,6 +289,7 @@ class UncountedCallArgsChecker
         Bug, "Call argument for 'this' parameter is uncounted and unsafe.",
         BSLoc);
     Report->addRange(CallArg->getSourceRange());
+    Report->setDeclWithIssue(DeclWithIssue);
     BR->emitReport(std::move(Report));
   }
 };
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
index 274da0baf2ce5..9d0a3bb5da732 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/UncountedLocalVarsChecker.cpp
@@ -121,6 +121,7 @@ class UncountedLocalVarsChecker
     // want to visit those, so we make our own RecursiveASTVisitor.
     struct LocalVisitor : public RecursiveASTVisitor<LocalVisitor> {
       const UncountedLocalVarsChecker *Checker;
+      Decl *DeclWithIssue{nullptr};
 
       TrivialFunctionAnalysis TFA;
 
@@ -134,10 +135,17 @@ class UncountedLocalVarsChecker
       bool shouldVisitTemplateInstantiations() const { return true; }
       bool shouldVisitImplicitCode() const { return false; }
 
+      bool TraverseDecl(Decl *D) {
+        llvm::SaveAndRestore SavedDecl(DeclWithIssue);
+        if (D && (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)))
+          DeclWithIssue = D;
+        return Base::TraverseDecl(D);
+      }
+
       bool VisitVarDecl(VarDecl *V) {
         auto *Init = V->getInit();
         if (Init && V->isLocalVarDecl())
-          Checker->visitVarDecl(V, Init);
+          Checker->visitVarDecl(V, Init, DeclWithIssue);
         return true;
       }
 
@@ -145,7 +153,7 @@ class UncountedLocalVarsChecker
         if (BO->isAssignmentOp()) {
           if (auto *VarRef = dyn_cast<DeclRefExpr>(BO->getLHS())) {
             if (auto *V = dyn_cast<VarDecl>(VarRef->getDecl()))
-              Checker->visitVarDecl(V, BO->getRHS());
+              Checker->visitVarDecl(V, BO->getRHS(), DeclWithIssue);
           }
         }
         return true;
@@ -186,7 +194,8 @@ class UncountedLocalVarsChecker
     visitor.TraverseDecl(const_cast<TranslationUnitDecl *>(TUD));
   }
 
-  void visitVarDecl(const VarDecl *V, const Expr *Value) const {
+  void visitVarDecl(const VarDecl *V, const Expr *Value,
+                    const Decl *DeclWithIssue) const {
     if (shouldSkipVarDecl(V))
       return;
 
@@ -240,7 +249,7 @@ class UncountedLocalVarsChecker
               }))
         return;
 
-      reportBug(V, Value);
+      reportBug(V, Value, DeclWithIssue);
     }
   }
 
@@ -249,7 +258,8 @@ class UncountedLocalVarsChecker
     return BR->getSourceManager().isInSystemHeader(V->getLocation());
   }
 
-  void reportBug(const VarDecl *V, const Expr *Value) const {
+  void reportBug(const VarDecl *V, const Expr *Value,
+                 const Decl *DeclWithIssue) const {
     assert(V);
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
@@ -278,6 +288,7 @@ class UncountedLocalVarsChecker
       PathDiagnosticLocation BSLoc(V->getLocation(), BR->getSourceManager());
       auto Report = std::make_unique<BasicBugReport>(Bug, Os.str(), BSLoc);
       Report->addRange(V->getSourceRange());
+      Report->setDeclWithIssue(DeclWithIssue);
       BR->emitReport(std::move(Report));
     }
   }
diff --git a/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
index 7cdd545e61b32..71268af22e242 100644
--- a/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/TextDiagnostics.cpp
@@ -91,6 +91,7 @@ class TextDiagnostics : public PathDiagnosticConsumer {
                                     ? " [" + PD->getCheckerName() + "]"
                                     : "")
                                    .str();
+
       reportPiece(WarnID, PD->getLocation().asLocation(),
                   (PD->getShortDescription() + WarningMsg).str(),
                   PD->path.back()->getRanges(), PD->path.back()->getFixits());
diff --git a/clang/lib/Testing/CommandLineArgs.cpp b/clang/lib/Testing/CommandLineArgs.cpp
index 3abc689b93e8d..88c6ce0e098bf 100644
--- a/clang/lib/Testing/CommandLineArgs.cpp
+++ b/clang/lib/Testing/CommandLineArgs.cpp
@@ -11,99 +11,79 @@
 #include "llvm/Support/ErrorHandling.h"
 
 namespace clang {
+std::vector<TestLanguage> getCOrLater(const int MinimumStd) {
+  std::vector<TestLanguage> Result{};
+
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  if (version >= MinimumStd)                                                   \
+    Result.push_back(Lang_##lang##version);
+#include "clang/Testing/TestLanguage.def"
+
+  return Result;
+}
+std::vector<TestLanguage> getCXXOrLater(const int MinimumStd) {
+  std::vector<TestLanguage> Result{};
+
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  if (version >= MinimumStd)                                                   \
+    Result.push_back(Lang_##lang##version);
+#include "clang/Testing/TestLanguage.def"
+
+  return Result;
+}
 
 std::vector<std::string> getCommandLineArgsForTesting(TestLanguage Lang) {
-  std::vector<std::string> Args;
   // Test with basic arguments.
   switch (Lang) {
-  case Lang_C89:
-    Args = {"-x", "c", "-std=c89"};
-    break;
-  case Lang_C99:
-    Args = {"-x", "c", "-std=c99"};
-    break;
-  case Lang_CXX03:
-    Args = {"-std=c++03", "-frtti"};
-    break;
-  case Lang_CXX11:
-    Args = {"-std=c++11", "-frtti"};
-    break;
-  case Lang_CXX14:
-    Args = {"-std=c++14", "-frtti"};
-    break;
-  case Lang_CXX17:
-    Args = {"-std=c++17", "-frtti"};
-    break;
-  case Lang_CXX20:
-    Args = {"-std=c++20", "-frtti"};
-    break;
-  case Lang_CXX23:
-    Args = {"-std=c++23", "-frtti"};
-    break;
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  case Lang_##lang##version:                                                   \
+    return { "-x", "c", "-std=" #std_flag };
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  case Lang_##lang##version:                                                   \
+    return { "-std=" #std_flag, "-frtti" };
+#include "clang/Testing/TestLanguage.def"
+
   case Lang_OBJC:
-    Args = {"-x", "objective-c", "-frtti", "-fobjc-nonfragile-abi"};
-    break;
+    return {"-x", "objective-c", "-frtti", "-fobjc-nonfragile-abi"};
   case Lang_OBJCXX:
-    Args = {"-x", "objective-c++", "-frtti"};
-    break;
+    return {"-x", "objective-c++", "-frtti"};
   case Lang_OpenCL:
-    llvm_unreachable("Not implemented yet!");
+    llvm_unreachable("Unhandled TestLanguage enum");
   }
-  return Args;
+  llvm_unreachable("Unhandled TestLanguage enum");
 }
 
 std::vector<std::string> getCC1ArgsForTesting(TestLanguage Lang) {
-  std::vector<std::string> Args;
   switch (Lang) {
-  case Lang_C89:
-    Args = {"-xc", "-std=c89"};
-    break;
-  case Lang_C99:
-    Args = {"-xc", "-std=c99"};
-    break;
-  case Lang_CXX03:
-    Args = {"-std=c++03"};
-    break;
-  case Lang_CXX11:
-    Args = {"-std=c++11"};
-    break;
-  case Lang_CXX14:
-    Args = {"-std=c++14"};
-    break;
-  case Lang_CXX17:
-    Args = {"-std=c++17"};
-    break;
-  case Lang_CXX20:
-    Args = {"-std=c++20"};
-    break;
-  case Lang_CXX23:
-    Args = {"-std=c++23"};
-    break;
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  case Lang_##lang##version:                                                   \
+    return { "-xc", "-std=" #std_flag };
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  case Lang_##lang##version:                                                   \
+    return { "-std=" #std_flag };
+#include "clang/Testing/TestLanguage.def"
+
   case Lang_OBJC:
-    Args = {"-xobjective-c"};
+    return {"-xobjective-c"};
     break;
   case Lang_OBJCXX:
-    Args = {"-xobjective-c++"};
+    return {"-xobjective-c++"};
     break;
   case Lang_OpenCL:
-    llvm_unreachable("Not implemented yet!");
+    llvm_unreachable("Unhandled TestLanguage enum");
   }
-  return Args;
+  llvm_unreachable("Unhandled TestLanguage enum");
 }
 
 StringRef getFilenameForTesting(TestLanguage Lang) {
   switch (Lang) {
-  case Lang_C89:
-  case Lang_C99:
+#define TESTLANGUAGE_C(lang, version, std_flag, version_index)                 \
+  case Lang_##lang##version:                                                   \
     return "input.c";
-
-  case Lang_CXX03:
-  case Lang_CXX11:
-  case Lang_CXX14:
-  case Lang_CXX17:
-  case Lang_CXX20:
-  case Lang_CXX23:
+#define TESTLANGUAGE_CXX(lang, version, std_flag, version_index)               \
+  case Lang_##lang##version:                                                   \
     return "input.cc";
+#include "clang/Testing/TestLanguage.def"
 
   case Lang_OpenCL:
     return "input.cl";
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 9fd5eae67a21f..18ccee382d44e 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -966,7 +966,8 @@ namespace shufflevector {
 namespace FunctionStart {
   void a(void) {}
   static_assert(__builtin_function_start(a) == a, ""); // both-error {{not an integral constant expression}} \
-                                                       // both-note {{comparison of addresses of literals has unspecified value}}
+                                                       // ref-note {{comparison against opaque constant address '&__builtin_function_start(a)'}} \
+                                                       // expected-note {{comparison of addresses of literals has unspecified value}}
 }
 
 namespace BuiltinInImplicitCtor {
diff --git a/clang/test/AST/ByteCode/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
index 68e212ff8933f..dea4055c531d2 100644
--- a/clang/test/AST/ByteCode/cxx20.cpp
+++ b/clang/test/AST/ByteCode/cxx20.cpp
@@ -99,7 +99,7 @@ constexpr int f() {
 static_assert(f());
 #endif
 
-/// Distinct literals have disctinct addresses.
+/// Distinct literals have distinct addresses.
 /// see https://github.com/llvm/llvm-project/issues/58754
 constexpr auto foo(const char *p) { return p; }
 constexpr auto p1 = "test1";
@@ -108,22 +108,16 @@ constexpr auto p2 = "test2";
 constexpr bool b1 = foo(p1) == foo(p1);
 static_assert(b1);
 
-constexpr bool b2 = foo(p1) == foo(p2); // ref-error {{must be initialized by a constant expression}} \
-                                        // ref-note {{comparison of addresses of literals}} \
-                                        // ref-note {{declared here}}
-static_assert(!b2); // ref-error {{not an integral constant expression}} \
-                    // ref-note {{not a constant expression}}
+constexpr bool b2 = foo(p1) == foo(p2);
+static_assert(!b2);
 
 constexpr auto name1() { return "name1"; }
 constexpr auto name2() { return "name2"; }
 
-constexpr auto b3 = name1() == name1();
-static_assert(b3);
-constexpr auto b4 = name1() == name2(); // ref-error {{must be initialized by a constant expression}} \
-                                        // ref-note {{has unspecified value}} \
-                                        // ref-note {{declared here}}
-static_assert(!b4); // ref-error {{not an integral constant expression}} \
-                    // ref-note {{not a constant expression}}
+constexpr auto b3 = name1() == name1(); // ref-error {{must be initialized by a constant expression}} \
+                                        // ref-note {{comparison of addresses of literals}}
+constexpr auto b4 = name1() == name2();
+static_assert(!b4);
 
 namespace UninitializedFields {
   class A {
diff --git a/clang/test/AST/ByteCode/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
index 756eec5b82560..1803fb8ab2e9a 100644
--- a/clang/test/AST/ByteCode/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -158,6 +158,21 @@ namespace VirtualBases {
     /// Calls the constructor of D.
     D d;
   }
+
+#if __cplusplus >= 202302L
+  struct VBase {};
+  struct HasVBase : virtual VBase {}; // all23-note 1{{virtual base class declared here}}
+  struct Derived : HasVBase {
+    constexpr Derived() {} // all23-error {{constexpr constructor not allowed in struct with virtual base class}}
+  };
+  template<typename T> struct DerivedFromVBase : T {
+    constexpr DerivedFromVBase();
+  };
+  constexpr int f(DerivedFromVBase<HasVBase>) {}
+  template<typename T> constexpr DerivedFromVBase<T>::DerivedFromVBase() : T() {}
+  constexpr int nVBase = (DerivedFromVBase<HasVBase>(), 0); // all23-error {{constant expression}} \
+                                                            // all23-note {{cannot construct object of type 'DerivedFromVBase<VirtualBases::HasVBase>' with virtual base class in a constant expression}}
+#endif
 }
 
 namespace LabelGoto {
@@ -238,3 +253,54 @@ namespace TwosComplementShifts {
   static_assert(-3 >> 1 == -2);
   static_assert(-7 >> 1 == -4);
 }
+
+namespace AnonUnionDtor {
+  struct A {
+    A ();
+    ~A();
+  };
+
+  template <class T>
+  struct opt
+  {
+    union { // all20-note {{is not literal}}
+      char c;
+      T data;
+    };
+
+    constexpr opt() {}
+
+    constexpr ~opt()  {
+     if (engaged)
+       data.~T();
+   }
+
+    bool engaged = false;
+  };
+
+  consteval void foo() {
+    opt<A> a; // all20-error {{variable of non-literal type}}
+  }
+
+  void bar() { foo(); }
+}
+
+/// FIXME: The two interpreters disagree about there to diagnose the non-constexpr destructor call.
+namespace NonLiteralDtorInParam {
+  class NonLiteral { // all20-note {{is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+  public:
+    NonLiteral() {}
+    ~NonLiteral() {} // all23-note {{declared here}}
+  };
+  constexpr int F2(NonLiteral N) { // all20-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} \
+                                   // ref23-note {{non-constexpr function '~NonLiteral' cannot be used in a constant expression}}
+    return 8;
+  }
+
+
+  void test() {
+    NonLiteral L;
+    constexpr auto D = F2(L); // all23-error {{must be initialized by a constant expression}} \
+                              // expected23-note {{non-constexpr function '~NonLiteral' cannot be used in a constant expression}}
+  }
+}
diff --git a/clang/test/AST/ByteCode/fixed-point.cpp b/clang/test/AST/ByteCode/fixed-point.cpp
new file mode 100644
index 0000000000000..bf1bd41783448
--- /dev/null
+++ b/clang/test/AST/ByteCode/fixed-point.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 %s -fsyntax-only -ffixed-point -verify=expected,both -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 %s -fsyntax-only -ffixed-point -verify=ref,both
+
+static_assert((bool)1.0k);
+static_assert(!((bool)0.0k));
+static_assert((bool)0.0k); // both-error {{static assertion failed}}
+
+static_assert(1.0k == 1.0k);
+static_assert(1.0k != 1.0k); // both-error {{failed due to requirement '1.0k != 1.0k'}}
+static_assert(-12.0k == -(-(-12.0k)));
+
+/// Zero-init.
+constexpr _Accum A{};
+static_assert(A == 0.0k);
diff --git a/clang/test/AST/ByteCode/placement-new.cpp b/clang/test/AST/ByteCode/placement-new.cpp
index 1ff6ff3ac1922..caf3ac97fd1c0 100644
--- a/clang/test/AST/ByteCode/placement-new.cpp
+++ b/clang/test/AST/ByteCode/placement-new.cpp
@@ -271,3 +271,18 @@ namespace ConstructAt {
                                           // both-note {{in call}}
 
 }
+
+namespace UsedToCrash {
+  struct S {
+      int* i;
+      constexpr S() : i(new int(42)) {} // #no-deallocation
+      constexpr ~S() {delete i;}
+  };
+  consteval void alloc() {
+      S* s = new S();
+      s->~S();
+      new (s) S();
+      delete s;
+  }
+  int alloc1 = (alloc(), 0);
+}
diff --git a/clang/test/AST/HLSL/RWBuffer-AST.hlsl b/clang/test/AST/HLSL/RWBuffer-AST.hlsl
index c3ba520e0f68e..a95be63da5dc1 100644
--- a/clang/test/AST/HLSL/RWBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/RWBuffer-AST.hlsl
@@ -66,7 +66,7 @@ RWBuffer<float> Buffer;
 // CHECK: TemplateArgument type 'float'
 // CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
 // CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float *
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'float *
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]] 
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
 // CHECK-SAME: ':'float *'
diff --git a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
index 1a3deba5830fa..a186779870c26 100644
--- a/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffer-AST.hlsl
@@ -70,7 +70,7 @@ StructuredBuffer<float> Buffer;
 // CHECK: TemplateArgument type 'float'
 // CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float'
 // CHECK-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> Implicit final
-// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float *
+// CHECK-NEXT: FieldDecl 0x{{[0-9A-Fa-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'float *
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
diff --git a/clang/test/Analysis/Checkers/WebKit/mock-types.h b/clang/test/Analysis/Checkers/WebKit/mock-types.h
index c427b22fd683e..933b4c5e62a79 100644
--- a/clang/test/Analysis/Checkers/WebKit/mock-types.h
+++ b/clang/test/Analysis/Checkers/WebKit/mock-types.h
@@ -108,4 +108,52 @@ struct RefCountable {
 
 template <typename T> T *downcast(T *t) { return t; }
 
+template <typename T> struct CheckedRef {
+private:
+  T *t;
+
+public:
+  CheckedRef() : t{} {};
+  CheckedRef(T &t) : t(t) { t->incrementPtrCount(); }
+  CheckedRef(const CheckedRef& o) : t(o.t) { if (t) t->incrementPtrCount(); }
+  ~CheckedRef() { if (t) t->decrementPtrCount(); }
+  T &get() { return *t; }
+  T *ptr() { return t; }
+  T *operator->() { return t; }
+  operator const T &() const { return *t; }
+  operator T &() { return *t; }
+};
+
+template <typename T> struct CheckedPtr {
+private:
+  T *t;
+
+public:
+  CheckedPtr() : t(nullptr) {}
+  CheckedPtr(T *t)
+    : t(t) {
+    if (t)
+      t->incrementPtrCount();
+  }
+  CheckedPtr(Ref<T>&& o)
+    : t(o.leakRef())
+  { }
+  ~CheckedPtr() {
+    if (t)
+      t->decrementPtrCount();
+  }
+  T *get() { return t; }
+  T *operator->() { return t; }
+  const T *operator->() const { return t; }
+  T &operator*() { return *t; }
+  CheckedPtr &operator=(T *) { return *this; }
+  operator bool() const { return t; }
+};
+
+class CheckedObj {
+public:
+  void incrementPtrCount();
+  void decrementPtrCount();
+};
+
 #endif
diff --git a/clang/test/Analysis/Checkers/WebKit/unchecked-members.cpp b/clang/test/Analysis/Checkers/WebKit/unchecked-members.cpp
new file mode 100644
index 0000000000000..0189b0cd50fcc
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/unchecked-members.cpp
@@ -0,0 +1,52 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.NoUncheckedPtrMemberChecker -verify %s
+
+#include "mock-types.h"
+
+namespace members {
+
+  struct Foo {
+  private:
+    CheckedObj* a = nullptr;
+// expected-warning@-1{{Member variable 'a' in 'members::Foo' is a raw pointer to CheckedPtr capable type 'CheckedObj'}}
+    CheckedObj& b;
+// expected-warning@-1{{Member variable 'b' in 'members::Foo' is a reference to CheckedPtr capable type 'CheckedObj'}}
+
+    [[clang::suppress]]
+    CheckedObj* a_suppressed = nullptr;
+
+    [[clang::suppress]]
+    CheckedObj& b_suppressed;
+
+    CheckedPtr<CheckedObj> c;
+    CheckedRef<CheckedObj> d;
+
+  public:
+    Foo();
+  };
+
+  template <typename S>
+  struct FooTmpl {
+    S* e;
+// expected-warning@-1{{Member variable 'e' in 'members::FooTmpl<CheckedObj>' is a raw pointer to CheckedPtr capable type 'CheckedObj'}}
+  };
+
+  void forceTmplToInstantiate(FooTmpl<CheckedObj>) { }
+
+} // namespace members
+
+namespace ignore_unions {
+
+  union Foo {
+    CheckedObj* a;
+    CheckedPtr<CheckedObj> c;
+    CheckedRef<CheckedObj> d;
+  };
+
+  template<class T>
+  union FooTmpl {
+    T* a;
+  };
+
+  void forceTmplToInstantiate(FooTmpl<CheckedObj>) { }
+
+} // namespace ignore_unions
diff --git a/clang/test/Analysis/Checkers/WebKit/uncounted-obj-const-v-muable.cpp b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-const-v-muable.cpp
new file mode 100644
index 0000000000000..2721cd8474e1b
--- /dev/null
+++ b/clang/test/Analysis/Checkers/WebKit/uncounted-obj-const-v-muable.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.webkit.UncountedCallArgsChecker -verify %s
+
+#include "mock-types.h"
+
+class Object {
+public:
+    void ref() const;
+    void deref() const;
+
+    bool constFunc() const;
+    void mutableFunc();
+};
+
+class Caller {
+  void someFunction();
+  void otherFunction();
+private:
+    RefPtr<Object> m_obj;
+};
+
+void Caller::someFunction()
+{
+    m_obj->constFunc();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+    m_obj->mutableFunc();
+    // expected-warning@-1{{Call argument for 'this' parameter is uncounted and unsafe}}
+}
diff --git a/clang/test/Analysis/Inputs/system-header-simulator.h b/clang/test/Analysis/Inputs/system-header-simulator.h
index 8fd51449ecc0a..fadc09f65d536 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator.h
@@ -154,3 +154,11 @@ void _Exit(int status) __attribute__ ((__noreturn__));
 #define EOF (-1)
 
 #define offsetof(t, d) __builtin_offsetof(t, d)
+
+struct sigaction {
+  void (*sa_handler)(int);
+};
+#define SIGINT 2
+#define	SIG_IGN (void (*)(int))1
+
+int sigaction(int, const struct sigaction *restrict, struct sigaction *restrict);
diff --git a/clang/test/Analysis/analyzer-config.c b/clang/test/Analysis/analyzer-config.c
index 8eb869bac46f8..47594e8317bc7 100644
--- a/clang/test/Analysis/analyzer-config.c
+++ b/clang/test/Analysis/analyzer-config.c
@@ -9,7 +9,6 @@
 // CHECK-NEXT: alpha.clone.CloneChecker:ReportNormalClones = true
 // CHECK-NEXT: alpha.cplusplus.STLAlgorithmModeling:AggressiveStdFindModeling = false
 // CHECK-NEXT: alpha.osx.cocoa.DirectIvarAssignment:AnnotatedFunctions = false
-// CHECK-NEXT: alpha.security.taint.TaintPropagation:Config = ""
 // CHECK-NEXT: apply-fixits = false
 // CHECK-NEXT: assume-controlled-environment = false
 // CHECK-NEXT: avoid-suppressing-null-argument-paths = false
@@ -111,6 +110,7 @@
 // CHECK-NEXT: optin.cplusplus.VirtualCall:ShowFixIts = false
 // CHECK-NEXT: optin.osx.cocoa.localizability.NonLocalizedStringChecker:AggressiveReport = false
 // CHECK-NEXT: optin.performance.Padding:AllowedPad = 24
+// CHECK-NEXT: optin.taint.TaintPropagation:Config = ""
 // CHECK-NEXT: osx.NumberObjectConversion:Pedantic = false
 // CHECK-NEXT: osx.cocoa.RetainCount:TrackNSCFStartParam = false
 // CHECK-NEXT: prune-paths = true
diff --git a/clang/test/Analysis/assume-controlled-environment.c b/clang/test/Analysis/assume-controlled-environment.c
index fce1a1e7bae33..4f663502f8657 100644
--- a/clang/test/Analysis/assume-controlled-environment.c
+++ b/clang/test/Analysis/assume-controlled-environment.c
@@ -1,12 +1,12 @@
 // RUN: %clang_analyze_cc1 -verify=untrusted-env %s \
 // RUN:   -analyzer-checker=core                    \
-// RUN:   -analyzer-checker=alpha.security.taint    \
+// RUN:   -analyzer-checker=optin.taint    \
 // RUN:   -analyzer-checker=debug.TaintTest
 
 // RUN: %clang_analyze_cc1 -verify %s -DEXPECT_NO_WARNINGS    \
 // RUN:   -analyzer-config assume-controlled-environment=true \
 // RUN:   -analyzer-checker=core                              \
-// RUN:   -analyzer-checker=alpha.security.taint              \
+// RUN:   -analyzer-checker=optin.taint              \
 // RUN:   -analyzer-checker=debug.TaintTest
 
 
diff --git a/clang/test/Analysis/bool-assignment.c b/clang/test/Analysis/bool-assignment.c
index c32bc8f9e8b14..3a104cf627ffa 100644
--- a/clang/test/Analysis/bool-assignment.c
+++ b/clang/test/Analysis/bool-assignment.c
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core.BoolAssignment,alpha.security.taint -verify -std=c99 -Dbool=_Bool %s
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core.BoolAssignment,alpha.security.taint -verify -x c++ %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core.BoolAssignment,optin.taint -verify -std=c99 -Dbool=_Bool %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core.BoolAssignment,optin.taint -verify -x c++ %s
 
 // Test C++'s bool and C's _Bool.
 // FIXME: We stopped warning on these when SValBuilder got smarter about
diff --git a/clang/test/Analysis/cxx-method-names.cpp b/clang/test/Analysis/cxx-method-names.cpp
index 22ec4db34796b..5254d82bd90b2 100644
--- a/clang/test/Analysis/cxx-method-names.cpp
+++ b/clang/test/Analysis/cxx-method-names.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,osx,alpha.unix,alpha.security.taint -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix,osx,alpha.unix,optin.taint -verify %s
 // expected-no-diagnostics
 
 class Evil {
diff --git a/clang/test/Analysis/debug-exprinspection-istainted.c b/clang/test/Analysis/debug-exprinspection-istainted.c
index 8d1ebca930885..b459f3a3e791b 100644
--- a/clang/test/Analysis/debug-exprinspection-istainted.c
+++ b/clang/test/Analysis/debug-exprinspection-istainted.c
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 -verify %s \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=debug.ExprInspection \
-// RUN:   -analyzer-checker=alpha.security.taint
+// RUN:   -analyzer-checker=optin.taint
 
 int scanf(const char *restrict format, ...);
 void clang_analyzer_isTainted(char);
diff --git a/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif b/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif
index 0ac96cacbed19..0bded6f0925d1 100644
--- a/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif
+++ b/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif
@@ -4,7 +4,7 @@
     {
       "artifacts": [
         {
-          "length": 434,
+          "length": 425,
           "location": {
             "index": 0,
           },
diff --git a/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif b/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif
index ae49ad6604cb2..7f9deea304832 100644
--- a/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif
+++ b/clang/test/Analysis/diagnostics/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif
@@ -4,7 +4,7 @@
     {
       "artifacts": [
         {
-          "length": 1081,
+          "length": 1071,
           "location": {
             "index": 0,
           },
diff --git a/clang/test/Analysis/diagnostics/sarif-diagnostics-taint-test.c b/clang/test/Analysis/diagnostics/sarif-diagnostics-taint-test.c
index b1042f9034d7d..7a3ca61c4319f 100644
--- a/clang/test/Analysis/diagnostics/sarif-diagnostics-taint-test.c
+++ b/clang/test/Analysis/diagnostics/sarif-diagnostics-taint-test.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.security.taint,debug.TaintTest %s -verify -analyzer-output=sarif -o - | %normalize_sarif | diff -U1 -b %S/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif -
+// RUN: %clang_analyze_cc1 -analyzer-checker=optin.taint,debug.TaintTest %s -verify -analyzer-output=sarif -o - | %normalize_sarif | diff -U1 -b %S/Inputs/expected-sarif/sarif-diagnostics-taint-test.c.sarif -
 #include "../Inputs/system-header-simulator.h"
 
 int atoi(const char *nptr);
diff --git a/clang/test/Analysis/diagnostics/sarif-multi-diagnostic-test.c b/clang/test/Analysis/diagnostics/sarif-multi-diagnostic-test.c
index 61d19817407e2..eeafd178628b3 100644
--- a/clang/test/Analysis/diagnostics/sarif-multi-diagnostic-test.c
+++ b/clang/test/Analysis/diagnostics/sarif-multi-diagnostic-test.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.security.taint,debug.TaintTest,unix.Malloc %s -verify -analyzer-output=sarif -o - | %normalize_sarif | diff -U1 -b %S/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif -
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,optin.taint,debug.TaintTest,unix.Malloc %s -verify -analyzer-output=sarif -o - | %normalize_sarif | diff -U1 -b %S/Inputs/expected-sarif/sarif-multi-diagnostic-test.c.sarif -
 #include "../Inputs/system-header-simulator.h"
 #include "../Inputs/system-header-simulator-for-malloc.h"
 #define ERR -1
@@ -43,4 +43,3 @@ int main(void) {
   unicode();
   return 0;
 }
-
diff --git a/clang/test/Analysis/fread.c b/clang/test/Analysis/fread.c
index 5dc6c0c744093..8dc998ea1e899 100644
--- a/clang/test/Analysis/fread.c
+++ b/clang/test/Analysis/fread.c
@@ -1,6 +1,6 @@
 // RUN: %clang_analyze_cc1 -verify %s \
 // RUN:   -triple x86_64-linux-gnu  \
-// RUN:   -analyzer-checker=core,unix.Stream,alpha.security.taint \
+// RUN:   -analyzer-checker=core,unix.Stream,optin.taint \
 // RUN:   -analyzer-checker=debug.ExprInspection
 
 #include "Inputs/system-header-simulator-for-simple-stream.h"
diff --git a/clang/test/Analysis/global-region-invalidation-errno.c b/clang/test/Analysis/global-region-invalidation-errno.c
index 9de10ad59095a..868869b5d262f 100644
--- a/clang/test/Analysis/global-region-invalidation-errno.c
+++ b/clang/test/Analysis/global-region-invalidation-errno.c
@@ -1,9 +1,9 @@
 // RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin10 -disable-free -verify %s \
-// RUN:   -analyzer-checker=core,deadcode,alpha.security.taint \
+// RUN:   -analyzer-checker=core,deadcode,optin.taint \
 // RUN:   -DERRNO_VAR
 
 // RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin10 -disable-free -verify %s \
-// RUN:   -analyzer-checker=core,deadcode,alpha.security.taint \
+// RUN:   -analyzer-checker=core,deadcode,optin.taint \
 // RUN:   -DERRNO_FUNC
 
 // Note, we do need to include headers here, since the analyzer checks if the function declaration is located in a system header.
diff --git a/clang/test/Analysis/global-region-invalidation.c b/clang/test/Analysis/global-region-invalidation.c
index faca3baf11caf..2f78467630ce6 100644
--- a/clang/test/Analysis/global-region-invalidation.c
+++ b/clang/test/Analysis/global-region-invalidation.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -triple x86_64-apple-darwin10 -disable-free -verify %s \
-// RUN:   -analyzer-checker=core,deadcode,alpha.security.taint,debug.TaintTest,debug.ExprInspection
+// RUN:   -analyzer-checker=core,deadcode,optin.taint,debug.TaintTest,debug.ExprInspection
 
 void clang_analyzer_eval(int);
 
diff --git a/clang/test/Analysis/malloc-annotations.c b/clang/test/Analysis/malloc-annotations.c
index c2fdf8a5641ae..c601a0383d221 100644
--- a/clang/test/Analysis/malloc-annotations.c
+++ b/clang/test/Analysis/malloc-annotations.c
@@ -3,6 +3,7 @@
 // RUN:   -analyzer-checker=alpha.deadcode.UnreachableCode \
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix.Malloc \
+// RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config unix.DynamicMemoryModeling:Optimistic=true %s
 
 typedef __typeof(sizeof(int)) size_t;
@@ -23,6 +24,12 @@ void __attribute((ownership_holds(malloc, 1))) my_hold(void *);
 void __attribute((ownership_holds(malloc, 1)))
 __attribute((ownership_holds(malloc, 1)))
 __attribute((ownership_holds(malloc, 3))) my_hold2(void *, void *, void *);
+
+__attribute((ownership_returns(user_malloc, 1))) void *user_malloc(size_t);
+__attribute((ownership_takes(user_malloc, 1))) void user_free(void *);
+
+void clang_analyzer_dump(int);
+
 void *my_malloc3(size_t);
 void *myglobalpointer;
 struct stuff {
@@ -273,3 +280,10 @@ void testMultipleFreeAnnotations(void) {
   my_freeBoth(p, q);
 }
 
+void testNoUninitAttr(void) {
+  int *p = user_malloc(sizeof(int));
+  int read = p[0]; // no-warning
+  clang_analyzer_dump(p[0]); // expected-warning{{Unknown}}
+  user_free(p);
+}
+
diff --git a/clang/test/Analysis/malloc.c b/clang/test/Analysis/malloc.c
index 9c7ca43bfbc5a..57f8561a74da6 100644
--- a/clang/test/Analysis/malloc.c
+++ b/clang/test/Analysis/malloc.c
@@ -4,7 +4,7 @@
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix \
 // RUN:   -analyzer-checker=debug.ExprInspection \
-// RUN:   -analyzer-checker=alpha.security.taint.TaintPropagation \
+// RUN:   -analyzer-checker=optin.taint.TaintPropagation \
 // RUN:   -analyzer-checker=optin.taint.TaintedAlloc
 
 #include "Inputs/system-header-simulator.h"
diff --git a/clang/test/Analysis/malloc.cpp b/clang/test/Analysis/malloc.cpp
index 7af1b59e04a5a..2bbfaf6640b79 100644
--- a/clang/test/Analysis/malloc.cpp
+++ b/clang/test/Analysis/malloc.cpp
@@ -4,7 +4,7 @@
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=cplusplus.NewDelete \
-// RUN:   -analyzer-checker=alpha.security.taint.TaintPropagation \
+// RUN:   -analyzer-checker=optin.taint.TaintPropagation \
 // RUN:   -analyzer-checker=optin.taint.TaintedAlloc
 
 // RUN: %clang_analyze_cc1 -w -verify %s \
@@ -14,7 +14,7 @@
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=cplusplus.NewDelete \
-// RUN:   -analyzer-checker=alpha.security.taint.TaintPropagation \
+// RUN:   -analyzer-checker=optin.taint.TaintPropagation \
 // RUN:   -analyzer-checker=optin.taint.TaintedAlloc
 
 // RUN: %clang_analyze_cc1 -w -verify %s -DTEST_INLINABLE_ALLOCATORS \
@@ -23,7 +23,7 @@
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=cplusplus.NewDelete \
-// RUN:   -analyzer-checker=alpha.security.taint.TaintPropagation \
+// RUN:   -analyzer-checker=optin.taint.TaintPropagation \
 // RUN:   -analyzer-checker=optin.taint.TaintedAlloc
 
 // RUN: %clang_analyze_cc1 -w -verify %s -DTEST_INLINABLE_ALLOCATORS \
@@ -33,7 +33,7 @@
 // RUN:   -analyzer-checker=alpha.core.CastSize \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=cplusplus.NewDelete \
-// RUN:   -analyzer-checker=alpha.security.taint.TaintPropagation \
+// RUN:   -analyzer-checker=optin.taint.TaintPropagation \
 // RUN:   -analyzer-checker=optin.taint.TaintedAlloc
 
 #include "Inputs/system-header-simulator-cxx.h"
diff --git a/clang/test/Analysis/out-of-bounds-diagnostics.c b/clang/test/Analysis/out-of-bounds-diagnostics.c
index de70e483add1c..8ecad7036c331 100644
--- a/clang/test/Analysis/out-of-bounds-diagnostics.c
+++ b/clang/test/Analysis/out-of-bounds-diagnostics.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
-// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,alpha.security.taint -verify %s
+// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,optin.taint -verify %s
 
 int TenElements[10];
 
diff --git a/clang/test/Analysis/out-of-bounds-notes.c b/clang/test/Analysis/out-of-bounds-notes.c
index c29b6f8ab111b..391089b6a35d8 100644
--- a/clang/test/Analysis/out-of-bounds-notes.c
+++ b/clang/test/Analysis/out-of-bounds-notes.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 -Wno-array-bounds -analyzer-output=text        \
-// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,alpha.security.taint -verify %s
+// RUN:     -analyzer-checker=core,alpha.security.ArrayBoundV2,unix.Malloc,optin.taint -verify %s
 
 int TenElements[10];
 
diff --git a/clang/test/Analysis/ptr-arith.c b/clang/test/Analysis/ptr-arith.c
index f99dfabb07366..020a500629230 100644
--- a/clang/test/Analysis/ptr-arith.c
+++ b/clang/test/Analysis/ptr-arith.c
@@ -1,6 +1,8 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple x86_64-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=alpha.core.FixedAddr,alpha.core.PointerArithm,debug.ExprInspection -Wno-pointer-to-int-cast -verify -triple i686-apple-darwin9 -Wno-tautological-pointer-compare -analyzer-config eagerly-assume=false %s
 
+#include "Inputs/system-header-simulator.h"
+
 void clang_analyzer_eval(int);
 void clang_analyzer_dump(int);
 
@@ -35,9 +37,20 @@ domain_port (const char *domain_b, const char *domain_e,
   return port;
 }
 
+#define FIXED_VALUE (int*) 0x1111
+
 void f4(void) {
   int *p;
   p = (int*) 0x10000; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
+  long x = 0x10100;
+  x += 10;
+  p = (int*) x; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
+
+  struct sigaction sa;
+  sa.sa_handler = SIG_IGN; // no warning (exclude macros defined in system header)
+  sigaction(SIGINT, &sa, NULL);
+
+  p = FIXED_VALUE; // expected-warning{{Using a fixed address is not portable because that address will probably not be valid in all environments or platforms}}
 }
 
 void f5(void) {
diff --git a/clang/test/Analysis/redefined_system.c b/clang/test/Analysis/redefined_system.c
index 0a55c36c6dd5b..0998fb92d2cd2 100644
--- a/clang/test/Analysis/redefined_system.c
+++ b/clang/test/Analysis/redefined_system.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=osx,unix,core,alpha.security.taint -w -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=osx,unix,core,optin.taint -w -verify %s
 // expected-no-diagnostics
 
 // Make sure we don't crash when someone redefines a system function we reason about.
diff --git a/clang/test/Analysis/string.c b/clang/test/Analysis/string.c
index 85232624160c0..79b4877eedbd9 100644
--- a/clang/test/Analysis/string.c
+++ b/clang/test/Analysis/string.c
@@ -25,7 +25,7 @@
 // RUN: %clang_analyze_cc1 -verify %s -Wno-null-dereference \
 // RUN:   -DUSE_BUILTINS -DVARIANT \
 // RUN:   -analyzer-checker=core \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint \
 // RUN:   -analyzer-checker=unix.cstring \
 // RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-checker=alpha.unix.cstring \
diff --git a/clang/test/Analysis/taint-checker-callback-order-has-definition.c b/clang/test/Analysis/taint-checker-callback-order-has-definition.c
index eaf96cc675f06..66c244c4cfda3 100644
--- a/clang/test/Analysis/taint-checker-callback-order-has-definition.c
+++ b/clang/test/Analysis/taint-checker-callback-order-has-definition.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s \
-// RUN:   -analyzer-checker=core,alpha.security.taint \
+// RUN:   -analyzer-checker=core,optin.taint \
 // RUN:   -mllvm -debug-only=taint-checker \
 // RUN:   2>&1 | FileCheck %s
 
diff --git a/clang/test/Analysis/taint-checker-callback-order-without-definition.c b/clang/test/Analysis/taint-checker-callback-order-without-definition.c
index 6de87f736926d..5f8df871b304d 100644
--- a/clang/test/Analysis/taint-checker-callback-order-without-definition.c
+++ b/clang/test/Analysis/taint-checker-callback-order-without-definition.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s \
-// RUN:   -analyzer-checker=core,alpha.security.taint \
+// RUN:   -analyzer-checker=core,optin.taint \
 // RUN:   -mllvm -debug-only=taint-checker \
 // RUN:   2>&1 | FileCheck %s
 
diff --git a/clang/test/Analysis/taint-diagnostic-visitor.c b/clang/test/Analysis/taint-diagnostic-visitor.c
index f51423646e8ae..526c04c360777 100644
--- a/clang/test/Analysis/taint-diagnostic-visitor.c
+++ b/clang/test/Analysis/taint-diagnostic-visitor.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.security.taint,core,alpha.security.ArrayBoundV2,optin.taint.TaintedAlloc -analyzer-output=text -verify %s
+// RUN: %clang_cc1 -analyze -analyzer-checker=optin.taint,core,alpha.security.ArrayBoundV2,optin.taint.TaintedAlloc -analyzer-output=text -verify %s
 
 // This file is for testing enhanced diagnostics produced by the GenericTaintChecker
 
diff --git a/clang/test/Analysis/taint-dumps.c b/clang/test/Analysis/taint-dumps.c
index 37fb6c2f2adf7..01bf0d7deff3a 100644
--- a/clang/test/Analysis/taint-dumps.c
+++ b/clang/test/Analysis/taint-dumps.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=alpha.security.taint\
+// RUN: %clang_analyze_cc1 -analyzer-checker=optin.taint\
 // RUN:                    -analyzer-checker=debug.ExprInspection %s\
 // RUN:                                          2>&1 | FileCheck %s
 
diff --git a/clang/test/Analysis/taint-generic.c b/clang/test/Analysis/taint-generic.c
index 1c139312734bc..a5cfdd9db1157 100644
--- a/clang/test/Analysis/taint-generic.c
+++ b/clang/test/Analysis/taint-generic.c
@@ -1,57 +1,57 @@
 // RUN: %clang_analyze_cc1 -Wno-format-security -Wno-pointer-to-int-cast \
 // RUN:   -Wno-incompatible-library-redeclaration -verify %s \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint.GenericTaint \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.security.ArrayBoundV2 \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
-// RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
+// RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
 
 // RUN: %clang_analyze_cc1 -Wno-format-security -Wno-pointer-to-int-cast \
 // RUN:   -Wno-incompatible-library-redeclaration -verify %s \
 // RUN:   -DFILE_IS_STRUCT \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint.GenericTaint \
 // RUN:   -analyzer-checker=core \
 // RUN:   -analyzer-checker=alpha.security.ArrayBoundV2 \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
-// RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
+// RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml
 
 // RUN: not %clang_analyze_cc1 -Wno-pointer-to-int-cast \
 // RUN:   -Wno-incompatible-library-redeclaration -verify %s \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint.GenericTaint  \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
-// RUN:     alpha.security.taint.TaintPropagation:Config=justguessit \
+// RUN:     optin.taint.TaintPropagation:Config=justguessit \
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-INVALID-FILE
 
 // CHECK-INVALID-FILE: (frontend): invalid input for checker option
-// CHECK-INVALID-FILE-SAME:        'alpha.security.taint.TaintPropagation:Config',
+// CHECK-INVALID-FILE-SAME:        'optin.taint.TaintPropagation:Config',
 // CHECK-INVALID-FILE-SAME:        that expects a valid filename instead of
 // CHECK-INVALID-FILE-SAME:        'justguessit'
 
 // RUN: not %clang_analyze_cc1 -Wno-incompatible-library-redeclaration \
 // RUN:   -verify %s \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint.GenericTaint  \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
-// RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-ill-formed.yaml \
+// RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-ill-formed.yaml \
 // RUN:   2>&1 | FileCheck -DMSG=%errc_EINVAL %s -check-prefix=CHECK-ILL-FORMED
 
 // CHECK-ILL-FORMED: (frontend): invalid input for checker option
-// CHECK-ILL-FORMED-SAME:        'alpha.security.taint.TaintPropagation:Config',
+// CHECK-ILL-FORMED-SAME:        'optin.taint.TaintPropagation:Config',
 // CHECK-ILL-FORMED-SAME:        that expects a valid yaml file: [[MSG]]
 
 // RUN: not %clang_analyze_cc1 -Wno-incompatible-library-redeclaration \
 // RUN:   -verify %s \
-// RUN:   -analyzer-checker=alpha.security.taint \
+// RUN:   -analyzer-checker=optin.taint.GenericTaint \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-config \
-// RUN:     alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-invalid-arg.yaml \
+// RUN:     optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config-invalid-arg.yaml \
 // RUN:   2>&1 | FileCheck %s -check-prefix=CHECK-INVALID-ARG
 
 // CHECK-INVALID-ARG: (frontend): invalid input for checker option
-// CHECK-INVALID-ARG-SAME:        'alpha.security.taint.TaintPropagation:Config',
+// CHECK-INVALID-ARG-SAME:        'optin.taint.TaintPropagation:Config',
 // CHECK-INVALID-ARG-SAME:        that expects an argument number for propagation
 // CHECK-INVALID-ARG-SAME:        rules greater or equal to -1
 
diff --git a/clang/test/Analysis/taint-generic.cpp b/clang/test/Analysis/taint-generic.cpp
index 0aadef88c704c..8092ac6f270b2 100644
--- a/clang/test/Analysis/taint-generic.cpp
+++ b/clang/test/Analysis/taint-generic.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1  -analyzer-checker=alpha.security.taint,core,alpha.security.ArrayBoundV2 -analyzer-config alpha.security.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml -Wno-format-security -verify -std=c++11 %s
+// RUN: %clang_analyze_cc1  -analyzer-checker=optin.taint,core,alpha.security.ArrayBoundV2 -analyzer-config optin.taint.TaintPropagation:Config=%S/Inputs/taint-generic-config.yaml -Wno-format-security -verify -std=c++11 %s
 
 #define BUFSIZE 10
 int Buffer[BUFSIZE];
diff --git a/clang/test/Analysis/taint-tester.c b/clang/test/Analysis/taint-tester.c
index 302349fb662dd..479a96c92ecec 100644
--- a/clang/test/Analysis/taint-tester.c
+++ b/clang/test/Analysis/taint-tester.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -Wno-int-to-pointer-cast -analyzer-checker=alpha.security.taint,debug.TaintTest %s -verify
+// RUN: %clang_analyze_cc1 -Wno-int-to-pointer-cast -analyzer-checker=optin.taint,debug.TaintTest %s -verify
 
 #include "Inputs/system-header-simulator.h"
 
diff --git a/clang/test/Analysis/taint-tester.cpp b/clang/test/Analysis/taint-tester.cpp
index 23a92cc56d248..a419938906800 100644
--- a/clang/test/Analysis/taint-tester.cpp
+++ b/clang/test/Analysis/taint-tester.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1  -analyzer-checker=alpha.security.taint,debug.TaintTest %s -verify
+// RUN: %clang_analyze_cc1  -analyzer-checker=optin.taint,debug.TaintTest %s -verify
 // expected-no-diagnostics
 
 typedef struct _FILE FILE;
@@ -32,4 +32,3 @@ void testOpaqueClass(opaque *obj) {
   char buf[20];
   snprintf(buf, 20, "%p", obj); // don't crash trying to load *obj
 }
-
diff --git a/clang/test/Analysis/taint-tester.m b/clang/test/Analysis/taint-tester.m
index 531c21b5faf88..3358a7769e257 100644
--- a/clang/test/Analysis/taint-tester.m
+++ b/clang/test/Analysis/taint-tester.m
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1  -analyzer-checker=alpha.security.taint,debug.TaintTest %s -verify
+// RUN: %clang_analyze_cc1  -analyzer-checker=optin.taint,debug.TaintTest %s -verify
 // expected-no-diagnostics
 
 #import <stdarg.h>
@@ -14,8 +14,8 @@ void TestLog (NSString *format, ...) {
     va_list ap;
     va_start(ap, format);
     NSString *string = @"AAA: ";
-    
+
     NSLogv([string stringByAppendingString:format], ap);
-    
+
     va_end(ap);
 }
\ No newline at end of file
diff --git a/clang/test/CodeGen/RISCV/riscv-cf-protection.c b/clang/test/CodeGen/RISCV/riscv-cf-protection.c
new file mode 100644
index 0000000000000..3a9855a3d2f01
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/riscv-cf-protection.c
@@ -0,0 +1,94 @@
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32i_zicfilp1p0 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=unlabeled -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,UNLABELED-FLAG %s
+
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32i_zicfilp1p0 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=func-sig -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32i_zicfilp1p0 -mcf-branch-label-scheme=unlabeled -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,UNLABELED-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN: -march=rv32i_zicfilp1p0 -mcf-branch-label-scheme=func-sig -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,FUNC-SIG-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv32 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=unlabeled -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,UNLABELED-FLAG %s
+
+// RUN: %clang --target=riscv32 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=func-sig -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// RUN: %clang --target=riscv32 -mcf-branch-label-scheme=unlabeled -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,UNLABELED-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv32 -mcf-branch-label-scheme=func-sig -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,FUNC-SIG-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64i_zicfilp1p0 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=unlabeled -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,UNLABELED-FLAG %s
+
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64i_zicfilp1p0 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=func-sig -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64i_zicfilp1p0 -mcf-branch-label-scheme=unlabeled -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,UNLABELED-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN: -march=rv64i_zicfilp1p0 -mcf-branch-label-scheme=func-sig -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,FUNC-SIG-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv64 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=unlabeled -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,UNLABELED-FLAG %s
+
+// RUN: %clang --target=riscv64 -fcf-protection=branch \
+// RUN: -mcf-branch-label-scheme=func-sig -S -emit-llvm %s -o - | FileCheck \
+// RUN: --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// RUN: %clang --target=riscv64 -mcf-branch-label-scheme=unlabeled -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,UNLABELED-SCHEME-UNUSED %s
+
+// RUN: %clang --target=riscv64 -mcf-branch-label-scheme=func-sig -S \
+// RUN: -emit-llvm %s -o - 2>&1 | FileCheck \
+// RUN: --check-prefixes=NO-FLAG,FUNC-SIG-SCHEME-UNUSED %s
+
+// Default -mcf-branch-label-scheme is func-sig
+// RUN: %clang --target=riscv32 -fcf-protection=branch -S -emit-llvm %s -o - \
+// RUN: | FileCheck --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// Default -mcf-branch-label-scheme is func-sig
+// RUN: %clang --target=riscv64 -fcf-protection=branch -S -emit-llvm %s -o - \
+// RUN: | FileCheck --check-prefixes=BRANCH-PROT-FLAG,FUNC-SIG-FLAG %s
+
+// UNLABELED-SCHEME-UNUSED: warning: argument unused during compilation:
+// UNLABELED-SCHEME-UNUSED-SAME: '-mcf-branch-label-scheme=unlabeled'
+// FUNC-SIG-SCHEME-UNUSED: warning: argument unused during compilation:
+// FUNC-SIG-SCHEME-UNUSED-SAME: '-mcf-branch-label-scheme=func-sig'
+
+// BRANCH-PROT-FLAG-DAG: [[P_FLAG:![0-9]+]] = !{i32 8, !"cf-protection-branch", i32 1}
+// UNLABELED-FLAG-DAG: [[S_FLAG:![0-9]+]] = !{i32 1, !"cf-branch-label-scheme", !"unlabeled"}
+// FUNC-SIG-FLAG-DAG: [[S_FLAG:![0-9]+]] = !{i32 1, !"cf-branch-label-scheme", !"func-sig"}
+// BRANCH-PROT-FLAG-DAG: !llvm.module.flags = !{{[{].*}}[[P_FLAG]]{{.*, }}[[S_FLAG]]{{(,.+)?[}]}}
+// NO-FLAG-NOT: !{i32 8, !"cf-protection-branch", i32 1}
+// NO-FLAG-NOT: !{i32 8, !"cf-branch-label-scheme", !"unlabeled"}
+// NO-FLAG-NOT: !{i32 8, !"cf-branch-label-scheme", !"func-sig"}
+
+int main() { return 0; }
diff --git a/clang/test/CodeGen/aarch64-fmv-dependencies.c b/clang/test/CodeGen/aarch64-fmv-dependencies.c
index f2eec11baed37..681f7e82634fa 100644
--- a/clang/test/CodeGen/aarch64-fmv-dependencies.c
+++ b/clang/test/CodeGen/aarch64-fmv-dependencies.c
@@ -111,9 +111,6 @@ __attribute__((target_version("rpres"))) int fmv(void) { return 0; }
 // CHECK: define dso_local i32 @fmv._Msb() #[[sb:[0-9]+]] {
 __attribute__((target_version("sb"))) int fmv(void) { return 0; }
 
-// CHECK: define dso_local i32 @fmv._Msha1() #[[ATTR0:[0-9]+]] {
-__attribute__((target_version("sha1"))) int fmv(void) { return 0; }
-
 // CHECK: define dso_local i32 @fmv._Msha2() #[[sha2:[0-9]+]] {
 __attribute__((target_version("sha2"))) int fmv(void) { return 0; }
 
diff --git a/clang/test/CodeGen/aarch64-neon-fp8-intrinsics/acle_neon_fscale.c b/clang/test/CodeGen/aarch64-neon-fp8-intrinsics/acle_neon_fscale.c
new file mode 100644
index 0000000000000..87fec3a491a2d
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-neon-fp8-intrinsics/acle_neon_fscale.c
@@ -0,0 +1,59 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4
+// REQUIRES: aarch64-registered-target
+#include <arm_neon.h>
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +fp8 -O3 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -target-feature +fp8 -S -O3 -o /dev/null %s
+
+// CHECK-LABEL: define dso_local <4 x half> @test_vscale_f16(
+// CHECK-SAME: <4 x half> noundef [[VN:%.*]], <4 x i16> noundef [[VM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FSCALE2_I:%.*]] = tail call <4 x half> @llvm.aarch64.neon.fp8.fscale.v4f16(<4 x half> [[VN]], <4 x i16> [[VM]])
+// CHECK-NEXT:    ret <4 x half> [[FSCALE2_I]]
+//
+float16x4_t test_vscale_f16(float16x4_t vn, int16x4_t vm) {
+  return vscale_f16(vn, vm);
+}
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vscaleq_f16(
+// CHECK-SAME: <8 x half> noundef [[VN:%.*]], <8 x i16> noundef [[VM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FSCALE2_I:%.*]] = tail call <8 x half> @llvm.aarch64.neon.fp8.fscale.v8f16(<8 x half> [[VN]], <8 x i16> [[VM]])
+// CHECK-NEXT:    ret <8 x half> [[FSCALE2_I]]
+//
+float16x8_t test_vscaleq_f16(float16x8_t vn, int16x8_t vm) {
+  return vscaleq_f16(vn, vm);
+
+}
+
+// CHECK-LABEL: define dso_local <2 x float> @test_vscale_f32(
+// CHECK-SAME: <2 x float> noundef [[VN:%.*]], <2 x i32> noundef [[VM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FSCALE2_I:%.*]] = tail call <2 x float> @llvm.aarch64.neon.fp8.fscale.v2f32(<2 x float> [[VN]], <2 x i32> [[VM]])
+// CHECK-NEXT:    ret <2 x float> [[FSCALE2_I]]
+//
+float32x2_t test_vscale_f32(float32x2_t vn, int32x2_t vm) {
+  return vscale_f32(vn, vm);
+
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vscaleq_f32(
+// CHECK-SAME: <4 x float> noundef [[VN:%.*]], <4 x i32> noundef [[VM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FSCALE2_I:%.*]] = tail call <4 x float> @llvm.aarch64.neon.fp8.fscale.v4f32(<4 x float> [[VN]], <4 x i32> [[VM]])
+// CHECK-NEXT:    ret <4 x float> [[FSCALE2_I]]
+//
+float32x4_t test_vscaleq_f32(float32x4_t vn, int32x4_t vm) {
+  return vscaleq_f32(vn, vm);
+
+}
+
+// CHECK-LABEL: define dso_local <2 x double> @test_vscale_f64(
+// CHECK-SAME: <2 x double> noundef [[VN:%.*]], <2 x i64> noundef [[VM:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[FSCALE2_I:%.*]] = tail call <2 x double> @llvm.aarch64.neon.fp8.fscale.v2f64(<2 x double> [[VN]], <2 x i64> [[VM]])
+// CHECK-NEXT:    ret <2 x double> [[FSCALE2_I]]
+//
+float64x2_t test_vscale_f64(float64x2_t vn, int64x2_t vm) {
+  return vscaleq_f64(vn, vm);
+}
diff --git a/clang/test/CodeGen/attr-target-version.c b/clang/test/CodeGen/attr-target-version.c
index 5dea8e9fe7903..228435a0494c3 100644
--- a/clang/test/CodeGen/attr-target-version.c
+++ b/clang/test/CodeGen/attr-target-version.c
@@ -24,7 +24,7 @@ int foo() {
   return fmv()+fmv_one()+fmv_two();
 }
 
-inline int __attribute__((target_version("sha1+pmull+f64mm"))) fmv_inline(void) { return 1; }
+inline int __attribute__((target_version("sha2+pmull+f64mm"))) fmv_inline(void) { return 1; }
 inline int __attribute__((target_version("fp16+fcma+rdma+sme+ fp16 "))) fmv_inline(void) { return 2; }
 inline int __attribute__((target_version("sha3+i8mm+f32mm"))) fmv_inline(void) { return 12; }
 inline int __attribute__((target_version("dit+sve-ebf16"))) fmv_inline(void) { return 8; }
@@ -659,7 +659,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 //
 //
 // CHECK: Function Attrs: noinline nounwind optnone
-// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha1
+// CHECK-LABEL: define {{[^@]+}}@fmv_inline._Mf64mmMpmullMsha2
 // CHECK-SAME: () #[[ATTR23:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    ret i32 1
@@ -805,12 +805,12 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK-NEXT:    ret ptr @fmv_inline._Msve2Msve2-bitpermMsve2-pmull128
 // CHECK:       resolver_else4:
 // CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 34359773184
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 34359773184
+// CHECK-NEXT:    [[TMP13:%.*]] = and i64 [[TMP12]], 34359775232
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 34359775232
 // CHECK-NEXT:    [[TMP15:%.*]] = and i1 true, [[TMP14]]
 // CHECK-NEXT:    br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
 // CHECK:       resolver_return5:
-// CHECK-NEXT:    ret ptr @fmv_inline._Mf64mmMpmullMsha1
+// CHECK-NEXT:    ret ptr @fmv_inline._Mf64mmMpmullMsha2
 // CHECK:       resolver_else6:
 // CHECK-NEXT:    [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
 // CHECK-NEXT:    [[TMP17:%.*]] = and i64 [[TMP16]], 17246986240
@@ -1135,7 +1135,7 @@ int caller(void) { return used_def_without_default_decl() + used_decl_without_de
 // CHECK: attributes #[[ATTR20]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+jsconv,+neon" }
 // CHECK: attributes #[[ATTR21:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+ls64" }
 // CHECK: attributes #[[ATTR22]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+sb" }
-// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sve" }
+// CHECK: attributes #[[ATTR23]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+aes,+f64mm,+fp-armv8,+fullfp16,+neon,+sha2,+sve" }
 // CHECK: attributes #[[ATTR24]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+fp-armv8,+fullfp16,+neon,+rdm,+sme" }
 // CHECK: attributes #[[ATTR25]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+f32mm,+fp-armv8,+fullfp16,+i8mm,+neon,+sha2,+sha3,+sve" }
 // CHECK: attributes #[[ATTR26]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+dit,+fp-armv8,+fullfp16,+neon,+sve" }
diff --git a/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp b/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
index 3d5e32516c7af..b967701ca1fa9 100644
--- a/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
+++ b/clang/test/CodeGenCXX/dynamic-cast-address-space.cpp
@@ -1,6 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --no-generate-body-for-unused-prefixes --version 4
 // RUN: %clang_cc1 -I%S %s -triple amdgcn-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s
-// RUN: %clang_cc1 -I%S %s -triple spirv64-unknown-unknown -fsycl-is-device -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 -I%S %s -triple spirv64-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 struct A { virtual void f(); };
 struct B : A { };
@@ -15,7 +15,7 @@ B fail;
 // CHECK: @_ZTI1B = linkonce_odr addrspace(1) constant { ptr addrspace(1), ptr addrspace(1), ptr addrspace(1) } { ptr addrspace(1) getelementptr inbounds (ptr addrspace(1), ptr addrspace(1) @_ZTVN10__cxxabiv120__si_class_type_infoE, i64 2), ptr addrspace(1) @_ZTS1B, ptr addrspace(1) @_ZTI1A }, comdat, align 8
 // CHECK: @__oclc_ABI_version = weak_odr hidden local_unnamed_addr addrspace(4) constant i32 500
 //.
-// WITH-NONZERO-DEFAULT-AS: @_ZTV1B = linkonce_odr unnamed_addr addrspace(1) constant { [3 x ptr addrspace(1)] } { [3 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) @_ZTI1B, ptr addrspace(1) addrspacecast (ptr @_ZN1A1fEv to ptr addrspace(1))] }, comdat, align 8
+// WITH-NONZERO-DEFAULT-AS: @_ZTV1B = linkonce_odr unnamed_addr addrspace(1) constant { [3 x ptr addrspace(1)] } { [3 x ptr addrspace(1)] [ptr addrspace(1) null, ptr addrspace(1) @_ZTI1B, ptr addrspace(1) addrspacecast (ptr addrspace(4) @_ZN1A1fEv to ptr addrspace(1))] }, comdat, align 8
 // WITH-NONZERO-DEFAULT-AS: @fail = addrspace(1) global { ptr addrspace(1) } { ptr addrspace(1) getelementptr inbounds inrange(-16, 8) ({ [3 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTV1B, i32 0, i32 0, i32 2) }, align 8
 // WITH-NONZERO-DEFAULT-AS: @_ZTI1A = external addrspace(1) constant ptr addrspace(1)
 // WITH-NONZERO-DEFAULT-AS: @_ZTVN10__cxxabiv120__si_class_type_infoE = external addrspace(1) global [0 x ptr addrspace(1)]
@@ -60,7 +60,7 @@ B fail;
 // CHECK-NEXT:    ret ptr addrspacecast (ptr addrspace(1) @fail to ptr)
 //
 // WITH-NONZERO-DEFAULT-AS-LABEL: define spir_func noundef align 8 dereferenceable(8) ptr addrspace(4) @_Z1fP1A(
-// WITH-NONZERO-DEFAULT-AS-SAME: ptr addrspace(4) noundef [[A:%.*]]) #[[ATTR0:[0-9]+]] personality ptr @__gxx_personality_v0 {
+// WITH-NONZERO-DEFAULT-AS-SAME: ptr addrspace(4) noundef [[A:%.*]]) addrspace(4) #[[ATTR0:[0-9]+]] personality ptr addrspace(4) @__gxx_personality_v0 {
 // WITH-NONZERO-DEFAULT-AS-NEXT:  entry:
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[RETVAL:%.*]] = alloca ptr addrspace(4), align 8
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[A_ADDR:%.*]] = alloca ptr addrspace(4), align 8
@@ -70,11 +70,11 @@ B fail;
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr [[A_ADDR]] to ptr addrspace(4)
 // WITH-NONZERO-DEFAULT-AS-NEXT:    store ptr addrspace(4) [[A]], ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP0:%.*]] = load ptr addrspace(4), ptr addrspace(4) [[A_ADDR_ASCAST]], align 8
-// WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP1:%.*]] = call spir_func ptr addrspace(4) @__dynamic_cast(ptr addrspace(4) [[TMP0]], ptr addrspace(1) @_ZTI1A, ptr addrspace(1) @_ZTI1B, i64 0) #[[ATTR3:[0-9]+]]
+// WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP1:%.*]] = call spir_func addrspace(4) ptr addrspace(4) @__dynamic_cast(ptr addrspace(4) [[TMP0]], ptr addrspace(1) @_ZTI1A, ptr addrspace(1) @_ZTI1B, i64 0) #[[ATTR3:[0-9]+]]
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP2:%.*]] = icmp eq ptr addrspace(4) [[TMP1]], null
 // WITH-NONZERO-DEFAULT-AS-NEXT:    br i1 [[TMP2]], label [[DYNAMIC_CAST_BAD_CAST:%.*]], label [[DYNAMIC_CAST_END:%.*]]
 // WITH-NONZERO-DEFAULT-AS:       dynamic_cast.bad_cast:
-// WITH-NONZERO-DEFAULT-AS-NEXT:    invoke spir_func void @__cxa_bad_cast() #[[ATTR4:[0-9]+]]
+// WITH-NONZERO-DEFAULT-AS-NEXT:    invoke spir_func addrspace(4) void @__cxa_bad_cast() #[[ATTR4:[0-9]+]]
 // WITH-NONZERO-DEFAULT-AS-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
 // WITH-NONZERO-DEFAULT-AS:       invoke.cont:
 // WITH-NONZERO-DEFAULT-AS-NEXT:    unreachable
@@ -90,8 +90,8 @@ B fail;
 // WITH-NONZERO-DEFAULT-AS-NEXT:    br label [[CATCH:%.*]]
 // WITH-NONZERO-DEFAULT-AS:       catch:
 // WITH-NONZERO-DEFAULT-AS-NEXT:    [[EXN:%.*]] = load ptr addrspace(4), ptr [[EXN_SLOT]], align 8
-// WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP6:%.*]] = call spir_func ptr addrspace(4) @__cxa_begin_catch(ptr addrspace(4) [[EXN]]) #[[ATTR3]]
-// WITH-NONZERO-DEFAULT-AS-NEXT:    call spir_func void @__cxa_end_catch()
+// WITH-NONZERO-DEFAULT-AS-NEXT:    [[TMP6:%.*]] = call spir_func addrspace(4) ptr addrspace(4) @__cxa_begin_catch(ptr addrspace(4) [[EXN]]) #[[ATTR3]]
+// WITH-NONZERO-DEFAULT-AS-NEXT:    call spir_func addrspace(4) void @__cxa_end_catch()
 // WITH-NONZERO-DEFAULT-AS-NEXT:    br label [[TRY_CONT]]
 // WITH-NONZERO-DEFAULT-AS:       try.cont:
 // WITH-NONZERO-DEFAULT-AS-NEXT:    ret ptr addrspace(4) addrspacecast (ptr addrspace(1) @fail to ptr addrspace(4))
@@ -112,9 +112,9 @@ const B& f(A *a) {
 // CHECK: attributes #[[ATTR3]] = { nounwind }
 // CHECK: attributes #[[ATTR4]] = { noreturn }
 //.
-// WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR0]] = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR0]] = { mustprogress noinline optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot11-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+fp8-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+gfx940-insts,+gws,+image-insts,+mai-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize32,+wavefrontsize64" }
 // WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR1:[0-9]+]] = { nounwind willreturn memory(read) }
-// WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR2:[0-9]+]] = { convergent nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+// WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR2:[0-9]+]] = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot11-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+fp8-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+gfx940-insts,+gws,+image-insts,+mai-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize32,+wavefrontsize64" }
 // WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR3]] = { nounwind }
 // WITH-NONZERO-DEFAULT-AS: attributes #[[ATTR4]] = { noreturn }
 //.
@@ -122,6 +122,7 @@ const B& f(A *a) {
 // CHECK: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
 // CHECK: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 //.
-// WITH-NONZERO-DEFAULT-AS: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
-// WITH-NONZERO-DEFAULT-AS: [[META1:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+// WITH-NONZERO-DEFAULT-AS: [[META0:![0-9]+]] = !{i32 1, !"amdhsa_code_object_version", i32 500}
+// WITH-NONZERO-DEFAULT-AS: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+// WITH-NONZERO-DEFAULT-AS: [[META2:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
 //.
diff --git a/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp
index 2487e0fcd4343..223e408a37892 100644
--- a/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp
+++ b/clang/test/CodeGenCXX/spirv-amdgcn-float16.cpp
@@ -7,22 +7,25 @@
 // CHECK-NEXT:    [[X:%.*]] = alloca half, align 2
 // CHECK-NEXT:    [[Y:%.*]] = alloca half, align 2
 // CHECK-NEXT:    [[Z:%.*]] = alloca half, align 2
-// CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr [[X]], align 2
-// CHECK-NEXT:    [[TMP1:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr [[X]] to ptr addrspace(4)
+// CHECK-NEXT:    [[Y_ASCAST:%.*]] = addrspacecast ptr [[Y]] to ptr addrspace(4)
+// CHECK-NEXT:    [[Z_ASCAST:%.*]] = addrspacecast ptr [[Z]] to ptr addrspace(4)
+// CHECK-NEXT:    [[TMP0:%.*]] = load half, ptr addrspace(4) [[X_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load half, ptr addrspace(4) [[Y_ASCAST]], align 2
 // CHECK-NEXT:    [[ADD:%.*]] = fadd half [[TMP0]], [[TMP1]]
-// CHECK-NEXT:    store half [[ADD]], ptr [[Z]], align 2
-// CHECK-NEXT:    [[TMP2:%.*]] = load half, ptr [[X]], align 2
-// CHECK-NEXT:    [[TMP3:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    store half [[ADD]], ptr addrspace(4) [[Z_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP2:%.*]] = load half, ptr addrspace(4) [[X_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load half, ptr addrspace(4) [[Y_ASCAST]], align 2
 // CHECK-NEXT:    [[SUB:%.*]] = fsub half [[TMP2]], [[TMP3]]
-// CHECK-NEXT:    store half [[SUB]], ptr [[Z]], align 2
-// CHECK-NEXT:    [[TMP4:%.*]] = load half, ptr [[X]], align 2
-// CHECK-NEXT:    [[TMP5:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    store half [[SUB]], ptr addrspace(4) [[Z_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP4:%.*]] = load half, ptr addrspace(4) [[X_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP5:%.*]] = load half, ptr addrspace(4) [[Y_ASCAST]], align 2
 // CHECK-NEXT:    [[MUL:%.*]] = fmul half [[TMP4]], [[TMP5]]
-// CHECK-NEXT:    store half [[MUL]], ptr [[Z]], align 2
-// CHECK-NEXT:    [[TMP6:%.*]] = load half, ptr [[X]], align 2
-// CHECK-NEXT:    [[TMP7:%.*]] = load half, ptr [[Y]], align 2
+// CHECK-NEXT:    store half [[MUL]], ptr addrspace(4) [[Z_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load half, ptr addrspace(4) [[X_ASCAST]], align 2
+// CHECK-NEXT:    [[TMP7:%.*]] = load half, ptr addrspace(4) [[Y_ASCAST]], align 2
 // CHECK-NEXT:    [[DIV:%.*]] = fdiv half [[TMP6]], [[TMP7]]
-// CHECK-NEXT:    store half [[DIV]], ptr [[Z]], align 2
+// CHECK-NEXT:    store half [[DIV]], ptr addrspace(4) [[Z_ASCAST]], align 2
 // CHECK-NEXT:    ret void
 //
 void f() {
diff --git a/clang/test/CodeGenCXX/template-param-objects-address-space.cpp b/clang/test/CodeGenCXX/template-param-objects-address-space.cpp
index b3733decdb550..1f7c160bb86e9 100644
--- a/clang/test/CodeGenCXX/template-param-objects-address-space.cpp
+++ b/clang/test/CodeGenCXX/template-param-objects-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -std=c++20 %s -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 -triple spirv64-unknown-unknown -fsycl-is-device -std=c++20 %s -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 -triple spirv64-amd-amdhsa -std=c++20 %s -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 struct S { char buf[32]; };
 template<S s> constexpr const char *begin() { return s.buf; }
@@ -37,6 +37,6 @@ const void *s = observable_addr<S{"hello world"}>();
 // CHECK: define linkonce_odr noundef ptr @_Z15observable_addrIXtl1StlA32_cLc104ELc101ELc108ELc108ELc111ELc32ELc119ELc111ELc114ELc108ELc100EEEEEPKvv()
 // WITH-NONZERO-DEFAULT-AS: define linkonce_odr {{.*}} noundef ptr addrspace(4) @_Z15observable_addrIXtl1StlA32_cLc104ELc101ELc108ELc108ELc111ELc32ELc119ELc111ELc114ELc108ELc100EEEEEPKvv()
 // CHECK: %call = call noundef ptr @_Z6calleePK1S(ptr noundef addrspacecast (ptr addrspace(1) [[HELLO]] to ptr))
-// WITH-NONZERO-DEFAULT-AS: %call = call {{.*}} noundef ptr addrspace(4) @_Z6calleePK1S(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) [[HELLO]] to ptr addrspace(4)))
+// WITH-NONZERO-DEFAULT-AS: %call = call {{.*}} noundef{{.*}} ptr addrspace(4) @_Z6calleePK1S(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) [[HELLO]] to ptr addrspace(4)))
 // CHECK: declare noundef ptr @_Z6calleePK1S(ptr noundef)
 // WITH-NONZERO-DEFAULT-AS: declare {{.*}} noundef ptr addrspace(4) @_Z6calleePK1S(ptr addrspace(4) noundef)
diff --git a/clang/test/CodeGenCXX/throw-expression-typeinfo-in-address-space.cpp b/clang/test/CodeGenCXX/throw-expression-typeinfo-in-address-space.cpp
index 3acbdd8fd97ee..0fb553de4485a 100644
--- a/clang/test/CodeGenCXX/throw-expression-typeinfo-in-address-space.cpp
+++ b/clang/test/CodeGenCXX/throw-expression-typeinfo-in-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -std=c++11 -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple spirv64-unknown-unknown -fsycl-is-device -emit-llvm -fcxx-exceptions -fexceptions -std=c++11 -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 %s -triple spirv64-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -std=c++11 -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 struct X {
   ~X();
diff --git a/clang/test/CodeGenCXX/try-catch-with-address-space.cpp b/clang/test/CodeGenCXX/try-catch-with-address-space.cpp
index 412ac6c287258..55c76ed7f344f 100644
--- a/clang/test/CodeGenCXX/try-catch-with-address-space.cpp
+++ b/clang/test/CodeGenCXX/try-catch-with-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple=amdgcn-amd-amdhsa -emit-llvm -o - -fcxx-exceptions -fexceptions | FileCheck %s
-// RUN: %clang_cc1 %s -triple=spirv64-unknown-unknown -fsycl-is-device -emit-llvm -o - -fcxx-exceptions -fexceptions | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 %s -triple=spirv64-amd-amdhsa -emit-llvm -o - -fcxx-exceptions -fexceptions | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 struct X { };
 
@@ -12,7 +12,7 @@ void f() {
   } catch (const X x) {
     // CHECK: catch ptr addrspace(1) @_ZTI1X
     // CHECK: call i32 @llvm.eh.typeid.for.p0(ptr addrspacecast (ptr addrspace(1) @_ZTI1X to ptr))
-    // WITH-NONZERO-DEFAULT-AS: call i32 @llvm.eh.typeid.for.p4(ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZTI1X to ptr addrspace(4)))
+    // WITH-NONZERO-DEFAULT-AS: call{{.*}} i32 @llvm.eh.typeid.for.p4(ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZTI1X to ptr addrspace(4)))
   }
 }
 
@@ -23,6 +23,6 @@ void h() {
   } catch (char const(&)[4]) {
     // CHECK: catch ptr addrspace(1) @_ZTIA4_c
     // CHECK: call i32 @llvm.eh.typeid.for.p0(ptr addrspacecast (ptr addrspace(1) @_ZTIA4_c to ptr))
-    // WITH-NONZERO-DEFAULT-AS: call i32 @llvm.eh.typeid.for.p4(ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZTIA4_c to ptr addrspace(4)))
+    // WITH-NONZERO-DEFAULT-AS: call{{.*}} i32 @llvm.eh.typeid.for.p4(ptr addrspace(4) addrspacecast (ptr addrspace(1) @_ZTIA4_c to ptr addrspace(4)))
   }
 }
diff --git a/clang/test/CodeGenCXX/typeid-cxx11-with-address-space.cpp b/clang/test/CodeGenCXX/typeid-cxx11-with-address-space.cpp
index f6dc38ec9f292..782e59c887bd3 100644
--- a/clang/test/CodeGenCXX/typeid-cxx11-with-address-space.cpp
+++ b/clang/test/CodeGenCXX/typeid-cxx11-with-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -I%S %s -triple amdgcn-amd-amdhsa -emit-llvm -std=c++11 -o - | FileCheck %s
-// RUN: %clang_cc1 -I%S %s -triple spirv64-unknown-unknown -fsycl-is-device -emit-llvm -std=c++11 -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 -I%S %s -triple spirv64-amd-amdhsa -emit-llvm -std=c++11 -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 #include <typeinfo>
 
 namespace Test1 {
diff --git a/clang/test/CodeGenCXX/typeid-with-address-space.cpp b/clang/test/CodeGenCXX/typeid-with-address-space.cpp
index 98af17f4fc888..20699dc170ace 100644
--- a/clang/test/CodeGenCXX/typeid-with-address-space.cpp
+++ b/clang/test/CodeGenCXX/typeid-with-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -I%S %s -triple amdgcn-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s
-// RUN: %clang_cc1 -I%S %s -triple spirv64-unknown-unknown -fsycl-is-device -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 -I%S %s -triple spirv64-amd-amdhsa -emit-llvm -fcxx-exceptions -fexceptions -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 #include <typeinfo>
 
 namespace Test1 {
@@ -39,7 +39,7 @@ const std::type_info &A10_c_ti = typeid(char const[10]);
 // CHECK-LABEL: define{{.*}} ptr @_ZN5Test11fEv
 // CHECK-SAME:  personality ptr @__gxx_personality_v0
 // WITH-NONZERO-DEFAULT-AS-LABEL: define{{.*}} ptr addrspace(4) @_ZN5Test11fEv
-// WITH-NONZERO-DEFAULT-AS-SAME:  personality ptr @__gxx_personality_v0
+// WITH-NONZERO-DEFAULT-AS-SAME: personality ptr addrspace(4) @__gxx_personality_v0
 const char *f() {
   try {
     // CHECK: br i1
diff --git a/clang/test/CodeGenCXX/typeinfo-with-address-space.cpp b/clang/test/CodeGenCXX/typeinfo-with-address-space.cpp
index 350303cc6e9b3..60eb8f17f91fd 100644
--- a/clang/test/CodeGenCXX/typeinfo-with-address-space.cpp
+++ b/clang/test/CodeGenCXX/typeinfo-with-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -I%S %s -triple amdgcn-amd-amdhsa -emit-llvm -o - | FileCheck %s -check-prefix=AS
-// RUN: %clang_cc1 -I%S %s -triple spirv64-unknown-unknown -fsycl-is-device -emit-llvm -o - | FileCheck %s -check-prefix=NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 -I%S %s -triple spirv64-amd-amdhsa -emit-llvm -o - | FileCheck %s -check-prefix=NONZERO-DEFAULT-AS
 // RUN: %clang_cc1 -I%S %s -triple x86_64-linux-gnu -emit-llvm -o - | FileCheck %s -check-prefix=NO-AS
 #include <typeinfo>
 
@@ -26,30 +26,30 @@ class B : A {
 
 unsigned long Fn(B& b) {
 // AS: %call = call noundef zeroext i1 @_ZNKSt9type_infoeqERKS_(ptr {{.*}} addrspacecast (ptr addrspace(1) @_ZTISt9type_info to ptr), ptr {{.*}} %2)
-// NONZERO-DEFAULT-AS: %call = call{{.*}} noundef zeroext i1 @_ZNKSt9type_infoeqERKS_(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTISt9type_info to ptr addrspace(4)), ptr addrspace(4) {{.*}} %2)
+// NONZERO-DEFAULT-AS: %call = call{{.*}} noundef zeroext{{.*}} i1 @_ZNKSt9type_infoeqERKS_(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTISt9type_info to ptr addrspace(4)), ptr addrspace(4) {{.*}} %2)
 // NO-AS: %call = call noundef zeroext i1 @_ZNKSt9type_infoeqERKS_(ptr {{.*}} @_ZTISt9type_info, ptr {{.*}} %2)
     if (typeid(std::type_info) == typeid(b))
         return 42;
 // AS: %call2 = call noundef zeroext i1 @_ZNKSt9type_infoneERKS_(ptr {{.*}} addrspacecast (ptr addrspace(1) @_ZTIi to ptr), ptr {{.*}} %5)
-// NONZERO-DEFAULT-AS: %call2 = call{{.*}} noundef zeroext i1 @_ZNKSt9type_infoneERKS_(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTIi to ptr addrspace(4)), ptr addrspace(4) {{.*}} %5)
+// NONZERO-DEFAULT-AS: %call2 = call{{.*}} noundef zeroext{{.*}} i1 @_ZNKSt9type_infoneERKS_(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTIi to ptr addrspace(4)), ptr addrspace(4) {{.*}} %5)
 // NO-AS: %call2 = call noundef zeroext i1 @_ZNKSt9type_infoneERKS_(ptr {{.*}} @_ZTIi, ptr {{.*}} %5)
     if (typeid(int) != typeid(b))
         return 1712;
 // AS: %call5 = call noundef ptr @_ZNKSt9type_info4nameEv(ptr {{.*}} addrspacecast (ptr addrspace(1) @_ZTI1A to ptr))
-// NONZERO-DEFAULT-AS: %call5 = call{{.*}} noundef ptr addrspace(4) @_ZNKSt9type_info4nameEv(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTI1A to ptr addrspace(4)))
+// NONZERO-DEFAULT-AS: %call5 = call{{.*}} noundef{{.*}} ptr addrspace(4) @_ZNKSt9type_info4nameEv(ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTI1A to ptr addrspace(4)))
 // NO-AS: %call5 = call noundef ptr @_ZNKSt9type_info4nameEv(ptr {{.*}} @_ZTI1A)
 // AS: %call7 = call noundef ptr @_ZNKSt9type_info4nameEv(ptr {{.*}} %8)
-// NONZERO-DEFAULT-AS: %call7 = call{{.*}} noundef ptr addrspace(4) @_ZNKSt9type_info4nameEv(ptr addrspace(4) {{.*}} %8)
+// NONZERO-DEFAULT-AS: %call7 = call{{.*}} noundef{{.*}} ptr addrspace(4) @_ZNKSt9type_info4nameEv(ptr addrspace(4) {{.*}} %8)
 // NO-AS: %call7 = call noundef ptr @_ZNKSt9type_info4nameEv(ptr {{.*}} %8)
     if (typeid(A).name() == typeid(b).name())
         return 0;
 // AS: %call11 = call noundef zeroext i1 @_ZNKSt9type_info6beforeERKS_(ptr {{.*}} %11, ptr {{.*}} addrspacecast (ptr addrspace(1) @_ZTIf to ptr))
-// NONZERO-DEFAULT-AS: %call11 = call{{.*}} noundef zeroext i1 @_ZNKSt9type_info6beforeERKS_(ptr addrspace(4) {{.*}} %11, ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTIf to ptr addrspace(4)))
+// NONZERO-DEFAULT-AS: %call11 = call{{.*}} noundef zeroext{{.*}} i1 @_ZNKSt9type_info6beforeERKS_(ptr addrspace(4) {{.*}} %11, ptr addrspace(4) {{.*}} addrspacecast (ptr addrspace(1) @_ZTIf to ptr addrspace(4)))
 // NO-AS:   %call11 = call noundef zeroext i1 @_ZNKSt9type_info6beforeERKS_(ptr {{.*}} %11, ptr {{.*}} @_ZTIf)
     if (typeid(b).before(typeid(float)))
         return 1;
 // AS: %call15 = call noundef i64 @_ZNKSt9type_info9hash_codeEv(ptr {{.*}} %14)
-// NONZERO-DEFAULT-AS: %call15 = call{{.*}} noundef i64 @_ZNKSt9type_info9hash_codeEv(ptr addrspace(4) {{.*}} %14)
+// NONZERO-DEFAULT-AS: %call15 = call{{.*}} noundef{{.*}} i64 @_ZNKSt9type_info9hash_codeEv(ptr addrspace(4) {{.*}} %14)
 // NO-AS: %call15 = call noundef i64 @_ZNKSt9type_info9hash_codeEv(ptr {{.*}} %14)
     return typeid(b).hash_code();
 }
diff --git a/clang/test/CodeGenCXX/vtable-assume-load-address-space.cpp b/clang/test/CodeGenCXX/vtable-assume-load-address-space.cpp
index ecafa99d8be00..e8d3a8407cda7 100644
--- a/clang/test/CodeGenCXX/vtable-assume-load-address-space.cpp
+++ b/clang/test/CodeGenCXX/vtable-assume-load-address-space.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 %s -triple=amdgcn-amd-amdhsa -std=c++11 -emit-llvm -o %t.ll -O1 -disable-llvm-passes -fms-extensions -fstrict-vtable-pointers
 // RUN: %clang_cc1 %s -triple i686-pc-win32 -emit-llvm -o %t.ms.ll -O1 -disable-llvm-passes -fms-extensions -fstrict-vtable-pointers
-// RUN: %clang_cc1 %s -triple=spirv64-unknown-unknown -fsycl-is-device -std=c++11 -emit-llvm -o %t.ll -O1 -disable-llvm-passes -fms-extensions -fstrict-vtable-pointers
+// RUN: %clang_cc1 %s -triple=spirv64-amd-amdhsa -std=c++11 -emit-llvm -o %t.ll -O1 -disable-llvm-passes -fms-extensions -fstrict-vtable-pointers
 // FIXME: Assume load should not require -fstrict-vtable-pointers
 
 // RUN: FileCheck --check-prefix=CHECK1 --input-file=%t.ll %s
@@ -29,7 +29,7 @@ void g(A *a) { a->foo(); }
 // CHECK1: call{{.*}} void @_ZN5test11AC1Ev(ptr {{((addrspace(4)){0,1})}}
 // CHECK1: %[[VTABLE:.*]] = load ptr addrspace(1), ptr {{((addrspace(4)){0,1})}}{{.*}}%{{.*}}
 // CHECK1: %[[CMP:.*]] = icmp eq ptr addrspace(1) %[[VTABLE]], getelementptr inbounds inrange(-16, 8) ({ [3 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test11AE, i32 0, i32 0, i32 2)
-// CHECK1: call void @llvm.assume(i1 %[[CMP]])
+// CHECK1: call{{.*}} void @llvm.assume(i1 %[[CMP]])
 // CHECK1-LABEL: {{^}}}
 
 void fooA() {
@@ -41,7 +41,7 @@ void fooA() {
 // CHECK1: call{{.*}} void @_ZN5test11BC1Ev(ptr {{[^,]*}} %{{.*}})
 // CHECK1: %[[VTABLE:.*]] = load ptr addrspace(1), ptr {{((addrspace(4)){0,1})}}{{.*}}%{{.*}}
 // CHECK1: %[[CMP:.*]] = icmp eq ptr addrspace(1) %[[VTABLE]], getelementptr inbounds inrange(-16, 8) ({ [3 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test11BE, i32 0, i32 0, i32 2)
-// CHECK1: call void @llvm.assume(i1 %[[CMP]])
+// CHECK1: call{{.*}} void @llvm.assume(i1 %[[CMP]])
 // CHECK1-LABEL: {{^}}}
 
 void fooB() {
@@ -75,12 +75,12 @@ void h(B *b) { b->bar(); }
 // CHECK2: call{{.*}} void @_ZN5test21CC1Ev(ptr
 // CHECK2: %[[VTABLE:.*]] = load ptr addrspace(1), ptr {{.*}}
 // CHECK2: %[[CMP:.*]] = icmp eq ptr addrspace(1) %[[VTABLE]], getelementptr inbounds inrange(-16, 8) ({ [3 x ptr addrspace(1)], [3 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test21CE, i32 0, i32 0, i32 2)
-// CHECK2: call void @llvm.assume(i1 %[[CMP]])
+// CHECK2: call{{.*}} void @llvm.assume(i1 %[[CMP]])
 
 // CHECK2: %[[ADD_PTR:.*]] = getelementptr inbounds i8, ptr {{((addrspace(4)){0,1})}}{{.*}}%{{.*}}, i64 8
 // CHECK2: %[[VTABLE2:.*]] = load ptr addrspace(1), ptr {{((addrspace(4)){0,1})}}{{.*}}%[[ADD_PTR]]
 // CHECK2: %[[CMP2:.*]] = icmp eq ptr addrspace(1) %[[VTABLE2]], getelementptr inbounds inrange(-16, 8) ({ [3 x ptr addrspace(1)], [3 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test21CE, i32 0, i32 1, i32 2)
-// CHECK2: call void @llvm.assume(i1 %[[CMP2]])
+// CHECK2: call{{.*}} void @llvm.assume(i1 %[[CMP2]])
 
 // CHECK2: call{{.*}} void @_ZN5test21gEPNS_1AE(
 // CHECK2-LABEL: {{^}}}
@@ -111,7 +111,7 @@ void g(B *a) { a->foo(); }
 // CHECK3-LABEL: define{{.*}} void @_ZN5test34testEv()
 // CHECK3: call{{.*}} void @_ZN5test31CC1Ev(ptr
 // CHECK3: %[[CMP:.*]] = icmp eq ptr addrspace(1) %{{.*}}, getelementptr inbounds inrange(-24, 8) ({ [4 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test31CE, i32 0, i32 0, i32 3)
-// CHECK3: call void @llvm.assume(i1 %[[CMP]])
+// CHECK3: call{{.*}} void @llvm.assume(i1 %[[CMP]])
 // CHECK3-LABLEL: }
 void test() {
   C c;
@@ -140,11 +140,11 @@ void g(C *c) { c->foo(); }
 // CHECK4: call{{.*}} void @_ZN5test41CC1Ev(ptr
 // CHECK4: %[[VTABLE:.*]] = load ptr addrspace(1), ptr {{((addrspace(4)){0,1})}}{{.*}}%{{.*}}
 // CHECK4: %[[CMP:.*]] = icmp eq ptr addrspace(1) %[[VTABLE]], getelementptr inbounds inrange(-32, 8) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test41CE, i32 0, i32 0, i32 4)
-// CHECK4: call void @llvm.assume(i1 %[[CMP]]
+// CHECK4: call{{.*}} void @llvm.assume(i1 %[[CMP]]
 
 // CHECK4: %[[VTABLE2:.*]] = load ptr addrspace(1), ptr {{((addrspace(4)){0,1})}}{{.*}}%{{.*}}
 // CHECK4: %[[CMP2:.*]] = icmp eq ptr addrspace(1) %[[VTABLE2]], getelementptr inbounds inrange(-32, 8) ({ [5 x ptr addrspace(1)] }, ptr addrspace(1) @_ZTVN5test41CE, i32 0, i32 0, i32 4)
-// CHECK4: call void @llvm.assume(i1 %[[CMP2]])
+// CHECK4: call{{.*}} void @llvm.assume(i1 %[[CMP2]])
 // CHECK4-LABEL: {{^}}}
 
 void test() {
@@ -214,7 +214,7 @@ void A::foo() {}
 
 // CHECK7-LABEL: define{{.*}} void @_ZN5test71gEv()
 // CHECK7: call{{.*}} void @_ZN5test71AC1Ev(
-// CHECK7: call void @llvm.assume(
+// CHECK7: call{{.*}} void @llvm.assume(
 // CHECK7-LABEL: {{^}}}
 void g() {
   A *a = new A();
@@ -257,7 +257,7 @@ struct E : A {
 };
 
 // CHECK8-LABEL: define{{.*}} void @_ZN5test81bEv()
-// CHECK8: call void @llvm.assume(
+// CHECK8: call{{.*}} void @llvm.assume(
 // CHECK8-LABEL: {{^}}}
 void b() {
   B b;
@@ -285,7 +285,7 @@ void d() {
 }
 
 // CHECK8-LABEL: define{{.*}} void @_ZN5test81eEv()
-// CHECK8: call void @llvm.assume(
+// CHECK8: call{{.*}} void @llvm.assume(
 // CHECK8-LABEL: {{^}}}
 void e() {
   E e;
diff --git a/clang/test/CodeGenCXX/vtable-pointer-initialization-address-space.cpp b/clang/test/CodeGenCXX/vtable-pointer-initialization-address-space.cpp
index 876d0845cc515..8b5b9dd353064 100644
--- a/clang/test/CodeGenCXX/vtable-pointer-initialization-address-space.cpp
+++ b/clang/test/CodeGenCXX/vtable-pointer-initialization-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple=amdgcn-amd-amdhsa -std=c++11 -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple=spirv64-unknown-unknown -fsycl-is-device -std=c++11 -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 %s -triple=spirv64-amd-amdhsa -std=c++11 -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 struct Field {
   Field();
diff --git a/clang/test/CodeGenCXX/vtt-address-space.cpp b/clang/test/CodeGenCXX/vtt-address-space.cpp
index 4c3d0a534611c..3409bc7cbb69d 100644
--- a/clang/test/CodeGenCXX/vtt-address-space.cpp
+++ b/clang/test/CodeGenCXX/vtt-address-space.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple=amdgcn-amd-amdhsa -std=c++11 -emit-llvm -o - | FileCheck %s
-// RUN: %clang_cc1 %s -triple=spirv64-unknown-unknown -fsycl-is-device -std=c++11 -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
+// RUN: %clang_cc1 %s -triple=spirv64-amd-amdhsa -std=c++11 -emit-llvm -o - | FileCheck %s --check-prefix=WITH-NONZERO-DEFAULT-AS
 
 // This is the sample from the C++ Itanium ABI, p2.6.2.
 namespace Test {
diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
index 174f4c3eaaad2..19699dcf14d9f 100644
--- a/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-constructor.hlsl
@@ -1,6 +1,12 @@
 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-SPIRV
 
+// XFAIL: *
+// This expectedly fails because create.handle is no longer called
+// from RWBuffer constructor and the replacement has not been
+// implemented yet. This test should be updated to expect
+// dx.create.handleFromBinding as part of issue #105076.
+
 RWBuffer<float> Buf;
 
 // CHECK: define linkonce_odr noundef ptr @"??0?$RWBuffer@M@hlsl@@QAA@XZ"
@@ -10,4 +16,4 @@ RWBuffer<float> Buf;
 // CHECK: store ptr %[[HandleRes]], ptr %h, align 4
 
 // CHECK-SPIRV: %[[HandleRes:[0-9]+]] = call ptr @llvm.spv.create.handle(i8 1)
-// CHECK-SPIRV: store ptr %[[HandleRes]], ptr %h, align 8
\ No newline at end of file
+// CHECK-SPIRV: store ptr %[[HandleRes]], ptr %h, align 8
diff --git a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
index 34019e5b18693..178332d03e640 100644
--- a/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/StructuredBuffer-constructor.hlsl
@@ -1,5 +1,12 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple spirv-vulkan-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s --check-prefix=CHECK-SPIRV
 
+// XFAIL: *
+// This expectedly fails because create.handle is no longer invoked
+// from StructuredBuffer constructor and the replacement has not been
+// implemented yet. This test should be updated to expect
+// dx.create.handleFromBinding as part of issue #105076.
+
 StructuredBuffer<float> Buf;
 
 // CHECK: define linkonce_odr noundef ptr @"??0?$StructuredBuffer@M@hlsl@@QAA@XZ"
diff --git a/clang/test/CodeGenHLSL/builtins/create_handle.hlsl b/clang/test/CodeGenHLSL/builtins/create_handle.hlsl
deleted file mode 100644
index 61226c2b54e72..0000000000000
--- a/clang/test/CodeGenHLSL/builtins/create_handle.hlsl
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
-
-void fn() {
-  (void)__builtin_hlsl_create_handle(0);
-}
-
-// CHECK: call ptr @llvm.dx.create.handle(i8 0)
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
index 138616ccca718..7f07160bf5e88 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
@@ -50,6 +50,10 @@ void test_s_wait_event_export_ready() {
 
 // CHECK-LABEL: @test_global_add_f32
 // CHECK: = atomicrmw fadd ptr addrspace(1) %addr, float %x syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
+#if !defined(__SPIRV__)
 void test_global_add_f32(float *rtn, global float *addr, float x) {
+#else
+void test_global_add_f32(float *rtn, __attribute__((address_space(1))) float *addr, float x) {
+#endif
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
index 9274c80abd8c0..bf5f2971cf118 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn.cl
@@ -665,23 +665,24 @@ void test_s_getpc(global ulong* out)
 }
 
 // CHECK-LABEL: @test_ds_append_lds(
-// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %ptr, i1 false)
-kernel void test_ds_append_lds(global int* out, local int* ptr) {
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.append.p3(ptr addrspace(3) %{{.+}}, i1 false)
 #if !defined(__SPIRV__)
-  *out = __builtin_amdgcn_ds_append(ptr);
+kernel void test_ds_append_lds(global int* out, local int* ptr) {
 #else
-  *out = __builtin_amdgcn_ds_append((__attribute__((address_space(3))) int*)(int*)ptr);
+kernel void test_ds_append_lds(__attribute__((address_space(1))) int* out, __attribute__((address_space(3))) int* ptr) {
 #endif
+  *out = __builtin_amdgcn_ds_append(ptr);
 }
 
 // CHECK-LABEL: @test_ds_consume_lds(
-// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %ptr, i1 false)
-kernel void test_ds_consume_lds(global int* out, local int* ptr) {
+// CHECK: {{.*}}call{{.*}} i32 @llvm.amdgcn.ds.consume.p3(ptr addrspace(3) %{{.+}}, i1 false)
+
 #if !defined(__SPIRV__)
-  *out = __builtin_amdgcn_ds_consume(ptr);
+kernel void test_ds_consume_lds(global int* out, local int* ptr) {
 #else
-  *out = __builtin_amdgcn_ds_consume((__attribute__((address_space(3))) int*)(int*)ptr);
+kernel void test_ds_consume_lds(__attribute__((address_space(1))) int* out, __attribute__((address_space(3))) int* ptr) {
 #endif
+  *out = __builtin_amdgcn_ds_consume(ptr);
 }
 
 // CHECK-LABEL: @test_gws_init(
@@ -835,7 +836,11 @@ kernel void test_s_setreg(uint val) {
 }
 
 // CHECK-LABEL test_atomic_inc_dec(
+#if !defined(__SPIRV__)
 void test_atomic_inc_dec(local uint *lptr, global uint *gptr, uint val) {
+#else
+void test_atomic_inc_dec(__attribute__((address_space(3))) uint *lptr, __attribute__((address_space(1))) uint *gptr, uint val) {
+#endif
   uint res;
 
   // CHECK: atomicrmw uinc_wrap ptr addrspace(3) %lptr, i32 %val syncscope("workgroup") seq_cst, align 4
@@ -851,7 +856,11 @@ void test_atomic_inc_dec(local uint *lptr, global uint *gptr, uint val) {
   res = __builtin_amdgcn_atomic_dec32(gptr, val, __ATOMIC_SEQ_CST, "");
 
   // CHECK: atomicrmw volatile udec_wrap ptr addrspace(1) %gptr, i32 %val seq_cst, align 4
+  #if !defined(__SPIRV__)
   res = __builtin_amdgcn_atomic_dec32((volatile global uint*)gptr, val, __ATOMIC_SEQ_CST, "");
+  #else
+  res = __builtin_amdgcn_atomic_dec32((volatile __attribute__((address_space(1))) uint*)gptr, val, __ATOMIC_SEQ_CST, "");
+  #endif
 }
 
 // CHECK-LABEL test_wavefrontsize(
diff --git a/clang/test/Driver/B-opt.c b/clang/test/Driver/B-opt.c
index df85dee4b7040..48139e71a9001 100644
--- a/clang/test/Driver/B-opt.c
+++ b/clang/test/Driver/B-opt.c
@@ -1,28 +1,28 @@
 // Check -B driver option.
 
 /// Target triple prefix is not detected for -B.
-// RUN: %clang %s -### -o %t.o -target i386-unknown-linux \
+// RUN: %clang %s -### -o %t.o --target=i386-unknown-linux \
 // RUN:     -B %S/Inputs/B_opt_tree/dir1 -fuse-ld=ld 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-B-OPT-TRIPLE %s
 // CHECK-B-OPT-TRIPLE-NOT: "{{.*}}/Inputs/B_opt_tree/dir1{{/|\\\\}}i386-unknown-linux-ld"
 //
-// RUN: %clang %s -### -o %t.o -target i386-unknown-linux \
+// RUN: %clang %s -### -o %t.o --target=i386-unknown-linux \
 // RUN:     -B %S/Inputs/B_opt_tree/dir2 -fuse-ld=ld 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-B-OPT-DIR %s
 // CHECK-B-OPT-DIR: "{{.*}}/Inputs/B_opt_tree/dir2{{/|\\\\}}ld"
 //
-// RUN: %clang %s -### -o %t.o -target i386-unknown-linux \
+// RUN: %clang %s -### -o %t.o --target=i386-unknown-linux \
 // RUN:     -B %S/Inputs/B_opt_tree/dir3/prefix- -fuse-ld=ld 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-B-OPT-PREFIX %s
 // CHECK-B-OPT-PREFIX: "{{.*}}/Inputs/B_opt_tree/dir3{{/|\\\\}}prefix-ld"
 //
-// RUN: %clang %s -### -o %t.o -target i386-unknown-linux \
+// RUN: %clang %s -### -o %t.o --target=i386-unknown-linux \
 // RUN:     -B %S/Inputs/B_opt_tree/dir3/prefix- \
 // RUN:     -B %S/Inputs/B_opt_tree/dir2 2>&1 -fuse-ld=ld \
 // RUN:   | FileCheck --check-prefix=CHECK-B-OPT-MULT %s
 // CHECK-B-OPT-MULT: "{{.*}}/Inputs/B_opt_tree/dir3{{/|\\\\}}prefix-ld"
 //
 // RUN: %clang -B %S/Inputs/does_not_exist -print-search-dirs \
-// RUN:     -target aarch64-linux-gnu \
+// RUN:     --target=aarch64-linux-gnu \
 // RUN:   | FileCheck --check-prefix=CHECK-B-OPT-INVALID %s
 // CHECK-B-OPT-INVALID-NOT: /..//bin
diff --git a/clang/test/Driver/as-options.s b/clang/test/Driver/as-options.s
index 73d002c7ef7ed..1e5f392de4555 100644
--- a/clang/test/Driver/as-options.s
+++ b/clang/test/Driver/as-options.s
@@ -1,36 +1,36 @@
 // PR21000: Test that -I is passed to both external and integrated assemblers.
 
-// RUN: %clang -target x86_64-linux-gnu -c -no-integrated-as %s \
+// RUN: %clang --target=x86_64-linux-gnu -c -no-integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target x86_64-linux-gnu -c -no-integrated-as %s \
+// RUN: %clang --target=x86_64-linux-gnu -c -no-integrated-as %s \
 // RUN:   -I foo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target x86_64-linux-gnu -c -integrated-as %s \
+// RUN: %clang --target=x86_64-linux-gnu -c -integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target x86_64-linux-gnu -c -integrated-as %s \
+// RUN: %clang --target=x86_64-linux-gnu -c -integrated-as %s \
 // RUN:   -I foo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
 // Other GNU targets
 
-// RUN: %clang -target aarch64-linux-gnu -c -no-integrated-as %s \
+// RUN: %clang --target=aarch64-linux-gnu -c -no-integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target aarch64-linux-gnu -c -integrated-as %s \
+// RUN: %clang --target=aarch64-linux-gnu -c -integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target armv7-linux-gnueabihf -c -no-integrated-as %s \
+// RUN: %clang --target=armv7-linux-gnueabihf -c -no-integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
-// RUN: %clang -target armv7-linux-gnueabihf -c -integrated-as %s \
+// RUN: %clang --target=armv7-linux-gnueabihf -c -integrated-as %s \
 // RUN:   -Ifoo_dir -### 2>&1 \
 // RUN:   | FileCheck %s
 
@@ -53,45 +53,45 @@
 // RUN:   -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=WARN --allow-empty %s
 
-// RUN: %clang -mimplicit-it=always -target armv7-linux-gnueabi -E \
+// RUN: %clang -mimplicit-it=always --target=armv7-linux-gnueabi -E \
 // RUN:   -fintegrated-as -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -mimplicit-it=always -target armv7-linux-gnueabi -E \
+// RUN: %clang -mimplicit-it=always --target=armv7-linux-gnueabi -E \
 // RUN:   -fno-integrated-as -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=WARN --allow-empty %s
 
-// RUN: %clang -mimplicit-it=always -target armv7-linux-gnueabi -E \
+// RUN: %clang -mimplicit-it=always --target=armv7-linux-gnueabi -E \
 // RUN:   -fintegrated-as -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -mimplicit-it=always -target armv7-linux-gnueabi -E \
+// RUN: %clang -mimplicit-it=always --target=armv7-linux-gnueabi -E \
 // RUN:   -fno-integrated-as -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=WARN --allow-empty %s
 
-// RUN: %clang -Wa,-mbig-obj -target i386-pc-windows -E -fintegrated-as \
+// RUN: %clang -Wa,-mbig-obj --target=i386-pc-windows -E -fintegrated-as \
 // RUN:   -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -Wa,-mbig-obj -target i386-pc-windows -E -fno-integrated-as \
+// RUN: %clang -Wa,-mbig-obj --target=i386-pc-windows -E -fno-integrated-as \
 // RUN:   -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
 
-// RUN: %clang -Wa,-mbig-obj -target i386-pc-windows -E -fintegrated-as \
+// RUN: %clang -Wa,-mbig-obj --target=i386-pc-windows -E -fintegrated-as \
 // RUN:   -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -Wa,-mbig-obj -target i386-pc-windows -E -fno-integrated-as \
+// RUN: %clang -Wa,-mbig-obj --target=i386-pc-windows -E -fno-integrated-as \
 // RUN:   -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
 
-// RUN: %clang -Xassembler -mbig-obj -target i386-pc-windows -E -fintegrated-as \
+// RUN: %clang -Xassembler -mbig-obj --target=i386-pc-windows -E -fintegrated-as \
 // RUN:   -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -Xassembler -mbig-obj -target i386-pc-windows -E \
+// RUN: %clang -Xassembler -mbig-obj --target=i386-pc-windows -E \
 // RUN:   -fno-integrated-as -o /dev/null -x c++ %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
 
-// RUN: %clang -Xassembler -mbig-obj -target i386-pc-windows -E -fintegrated-as \
+// RUN: %clang -Xassembler -mbig-obj --target=i386-pc-windows -E -fintegrated-as \
 // RUN:   -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
-// RUN: %clang -Xassembler -mbig-obj -target i386-pc-windows -E \
+// RUN: %clang -Xassembler -mbig-obj --target=i386-pc-windows -E \
 // RUN:   -fno-integrated-as -o /dev/null -x assembler-with-cpp %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=NOWARN --allow-empty %s
 
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index a6f338533ad76..07a25da0269fd 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -406,9 +406,9 @@
 // RUN:    /Zm \
 // RUN:    /Zo \
 // RUN:    /Zo- \
-// RUN:    -### -- %s 2>&1 | FileCheck -check-prefix=IGNORED %s
+// RUN:    -### -- %s 2>&1 | FileCheck -DMSG=%errc_ENOENT -check-prefix=IGNORED %s
 // IGNORED-NOT: argument unused during compilation
-// IGNORED-NOT: no such file or directory
+// IGNORED-NOT: [[MSG]]
 // Don't confuse /openmp- with the /o flag:
 // IGNORED-NOT: "-o" "penmp-.obj"
 
diff --git a/clang/test/Driver/cl-zc.cpp b/clang/test/Driver/cl-zc.cpp
index c7cf5b1b6525b..9b1ea53888ceb 100644
--- a/clang/test/Driver/cl-zc.cpp
+++ b/clang/test/Driver/cl-zc.cpp
@@ -133,9 +133,9 @@
 // RUN:   /Zc:inline \
 // RUN:   /Zc:rvalueCast \
 // RUN:   /Zc:ternary \
-// RUN:   -### -- %s 2>&1 | FileCheck -check-prefix=IGNORED %s
+// RUN:   -### -- %s 2>&1 | FileCheck -DMSG=%errc_ENOENT -check-prefix=IGNORED %s
 // IGNORED-NOT: argument unused during compilation
-// IGNORED-NOT: no such file or directory
+// IGNORED-NOT: [[MSG]]
 
 // Negated form warns:
 // RUN: %clang_cl /c \
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index adb6f075b6c15..2e1736ca72aa7 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -463,8 +463,8 @@
 // RUN: %clang -### -S -fno-unsigned-char %s 2>&1 | FileCheck -check-prefix=CHAR-SIGN4 %s
 // CHAR-SIGN4-NOT: -fno-signed-char
 
-// RUN: %clang -target x86_64-unknown-none-none -### -fshort-wchar -fno-short-wchar %s 2>&1 | FileCheck -check-prefix=CHECK-WCHAR1 -check-prefix=DELIMITERS %s
-// RUN: %clang -target x86_64-unknown-none-none -### -fno-short-wchar -fshort-wchar %s 2>&1 | FileCheck -check-prefix=CHECK-WCHAR2 -check-prefix=DELIMITERS %s
+// RUN: %clang --target=x86_64-unknown-none-none -### -fshort-wchar -fno-short-wchar %s 2>&1 | FileCheck -check-prefix=CHECK-WCHAR1 -check-prefix=DELIMITERS %s
+// RUN: %clang --target=x86_64-unknown-none-none -### -fno-short-wchar -fshort-wchar %s 2>&1 | FileCheck -check-prefix=CHECK-WCHAR2 -check-prefix=DELIMITERS %s
 // Make sure we don't match the -NOT lines with the linker invocation.
 // Delimiters match the start of the cc1 and the start of the linker lines
 // DELIMITERS: {{^ (\(in-process\)|")}}
@@ -489,7 +489,7 @@
 // CHECK-ALLOW-PLACEHOLDERS: -fallow-editor-placeholders
 // CHECK-NO-ALLOW-PLACEHOLDERS-NOT: -fallow-editor-placeholders
 
-// RUN: %clang -### -target x86_64-unknown-windows-msvc -fno-short-wchar %s 2>&1 | FileCheck -check-prefix CHECK-WINDOWS-ISO10646 %s
+// RUN: %clang -### --target=x86_64-unknown-windows-msvc -fno-short-wchar %s 2>&1 | FileCheck -check-prefix CHECK-WINDOWS-ISO10646 %s
 // CHECK-WINDOWS-ISO10646: "-fwchar-type=int"
 // CHECK-WINDOWS-ISO10646: "-fsigned-wchar"
 
@@ -530,16 +530,16 @@
 // CHECK-NO-NULL-POINTER-CHECKS: "-fno-delete-null-pointer-checks"
 // CHECK-NULL-POINTER-CHECKS-NOT: "-fno-delete-null-pointer-checks"
 
-// RUN: %clang -### -S -target x86_64-unknown-linux -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -fno-record-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -fno-record-gcc-switches -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -frecord-gcc-switches -fno-record-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -fno-record-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -fno-record-command-line -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
-// RUN: %clang -### -S -target x86_64-unknown-linux -frecord-command-line -fno-record-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -fno-record-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -fno-record-gcc-switches -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -frecord-gcc-switches -fno-record-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -fno-record-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -fno-record-command-line -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-linux -frecord-command-line -fno-record-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-NO-RECORD-GCC-SWITCHES %s
 // Test with a couple examples of non-ELF object file formats
-// RUN: %clang -### -S -target x86_64-unknown-macosx -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
+// RUN: %clang -### -S --target=x86_64-unknown-macosx -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES %s
 // RUN: not %clang -### -S --target=x86_64-unknown-windows -frecord-command-line %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES-ERROR %s
 // CHECK-RECORD-GCC-SWITCHES: "-record-command-line"
 // CHECK-NO-RECORD-GCC-SWITCHES-NOT: "-record-command-line"
@@ -553,7 +553,7 @@
 // RUN: rm -rf "%t.r/with spaces"
 // RUN: mkdir -p "%t.r/with spaces"
 // RUN: cp %clang "%t.r/with spaces/clang"
-// RUN: "%t.r/with spaces/clang" -### -S -target x86_64-unknown-linux -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES-ESCAPED %s
+// RUN: "%t.r/with spaces/clang" -### -S --target=x86_64-unknown-linux -frecord-gcc-switches %s 2>&1 | FileCheck -check-prefix=CHECK-RECORD-GCC-SWITCHES-ESCAPED %s
 // CHECK-RECORD-GCC-SWITCHES-ESCAPED: "-record-command-line" "{{.+}}with\\ spaces{{.+}}"
 // Clean up copy of large binary copied into temp directory to avoid bloat.
 // RUN: rm -f "%t.r/with spaces/clang" || true
@@ -599,15 +599,15 @@
 // CHECK_DISABLE_DIRECT: -fobjc-disable-direct-methods-for-testing
 // CHECK_NO_DISABLE_DIRECT-NOT: -fobjc-disable-direct-methods-for-testing
 
-// RUN: %clang -### -S -fjmc -target x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefixes=CHECK_JMC_WARN,CHECK_NOJMC %s
-// RUN: %clang -### -S -fjmc -target x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefixes=CHECK_JMC_WARN,CHECK_NOJMC %s
-// RUN: %clang -### -S -fjmc -g -target x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_JMC %s
-// RUN: %clang -### -S -fjmc -g -fno-jmc -target x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
-// RUN: %clang -### -S -fjmc -g -target x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC %s
-// RUN: %clang -### -S -fjmc -g -fno-jmc -target x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
-// RUN: %clang -### -fjmc -g -flto -target x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
-// RUN: %clang -### -fjmc -g -flto -target x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC_LTO %s
-// RUN: %clang -### -fjmc -g -flto -fno-jmc -target x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
+// RUN: %clang -### -S -fjmc --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefixes=CHECK_JMC_WARN,CHECK_NOJMC %s
+// RUN: %clang -### -S -fjmc --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefixes=CHECK_JMC_WARN,CHECK_NOJMC %s
+// RUN: %clang -### -S -fjmc -g --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_JMC %s
+// RUN: %clang -### -S -fjmc -g -fno-jmc --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
+// RUN: %clang -### -S -fjmc -g --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC %s
+// RUN: %clang -### -S -fjmc -g -fno-jmc --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC %s
+// RUN: %clang -### -fjmc -g -flto --target=x86_64-pc-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
+// RUN: %clang -### -fjmc -g -flto --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_JMC_LTO %s
+// RUN: %clang -### -fjmc -g -flto -fno-jmc --target=x86_64-unknown-linux %s 2>&1 | FileCheck -check-prefix=CHECK_NOJMC_LTO %s
 // CHECK_JMC_WARN: -fjmc requires debug info. Use -g or debug options that enable debugger's stepping function; option ignored
 // CHECK_JMC_WARN_NOT_ELF: -fjmc works only for ELF; option ignored
 // CHECK_NOJMC-NOT: -fjmc
@@ -615,7 +615,7 @@
 // CHECK_NOJMC_LTO-NOT: -plugin-opt=-enable-jmc-instrument
 // CHECK_JMC_LTO: -plugin-opt=-enable-jmc-instrument
 
-// RUN: %clang -### -fintegrated-objemitter -target x86_64 %s 2>&1 | FileCheck -check-prefix=CHECK-INT-OBJEMITTER %s
+// RUN: %clang -### -fintegrated-objemitter --target=x86_64 %s 2>&1 | FileCheck -check-prefix=CHECK-INT-OBJEMITTER %s
 // CHECK-INT-OBJEMITTER-NOT: unsupported option '-fintegrated-objemitter' for target
 // RUN: not %clang -### -fno-integrated-objemitter --target=x86_64 %s 2>&1 | FileCheck -check-prefix=CHECK-NOINT-OBJEMITTER %s
 // CHECK-NOINT-OBJEMITTER: unsupported option '-fno-integrated-objemitter' for target
diff --git a/clang/test/Driver/config-file-errs.c b/clang/test/Driver/config-file-errs.c
index 96b49b2acf8ab..dc4fcdebf44bc 100644
--- a/clang/test/Driver/config-file-errs.c
+++ b/clang/test/Driver/config-file-errs.c
@@ -6,13 +6,13 @@
 
 //--- Argument of '--config' must be existing file, if it is specified by path.
 //
-// RUN: not %clang --config somewhere/nonexistent-config-file 2>&1 | FileCheck %s -check-prefix CHECK-NONEXISTENT
-// CHECK-NONEXISTENT: configuration file '{{.*}}somewhere{{.}}nonexistent-config-file' cannot be opened: {{[Nn]}}o such file or directory
+// RUN: not %clang --config somewhere/nonexistent-config-file 2>&1 | FileCheck -DMSG=%errc_ENOENT %s -check-prefix CHECK-NONEXISTENT
+// CHECK-NONEXISTENT: configuration file '{{.*}}somewhere{{.}}nonexistent-config-file' cannot be opened: [[MSG]]
 
 
 //--- All '--config' arguments must be existing files.
 //
-// RUN: not %clang --config %S/Inputs/config-4.cfg --config somewhere/nonexistent-config-file 2>&1 | FileCheck %s -check-prefix CHECK-NONEXISTENT
+// RUN: not %clang --config %S/Inputs/config-4.cfg --config somewhere/nonexistent-config-file 2>&1 | FileCheck -DMSG=%errc_ENOENT %s -check-prefix CHECK-NONEXISTENT
 
 
 //--- Argument of '--config' must exist somewhere in well-known directories, if it is specified by bare name.
diff --git a/clang/test/Driver/fbasic-block-sections.c b/clang/test/Driver/fbasic-block-sections.c
index e13cc81e910be..6dfba5f404cee 100644
--- a/clang/test/Driver/fbasic-block-sections.c
+++ b/clang/test/Driver/fbasic-block-sections.c
@@ -22,7 +22,8 @@
 // CHECK-OPT-NONE:   "-fbasic-block-sections=none"
 // CHECK-OPT-ALL:    "-fbasic-block-sections=all"
 // CHECK-OPT-LIST:   "-fbasic-block-sections={{[^ ]*}}fbasic-block-sections.c"
-// CHECK-OPT-LABELS: "-fbasic-block-sections=labels"
+// CHECK-OPT-LABELS: warning: argument '-fbasic-block-sections=labels' is deprecated, use '-fbasic-block-address-map' instead
+// CHECK-OPT-LABELS: "-fbasic-block-address-map"
 // CHECK-TRIPLE:     error: unsupported option '-fbasic-block-sections=all' for target
 // CHECK-INVALID-VALUE: error: invalid value {{[^ ]*}} in '-fbasic-block-sections={{.*}}'
 // CHECK-OPT-NULL-LIST: "-fbasic-block-sections=list="
diff --git a/clang/test/Driver/notypecheck.s b/clang/test/Driver/notypecheck.s
index f6e78d6791182..8e924b57fbdc0 100644
--- a/clang/test/Driver/notypecheck.s
+++ b/clang/test/Driver/notypecheck.s
@@ -5,7 +5,7 @@
 
 # Verify that without -Wa,--no-type-check the assembler will error out
 # RUN: not %clang %s -c -o tmp.o -target wasm32-unknown-unknown 2>&1 | FileCheck --check-prefix=ERROR %s
-# ERROR: error: popped i64, expected i32
+# ERROR: error: type mismatch, expected [i32] but got [i64]
 
 foo:
   .functype  foo () -> (i32)
diff --git a/clang/test/Driver/relax.s b/clang/test/Driver/relax.s
index b4a696a328eb5..0768a38834447 100644
--- a/clang/test/Driver/relax.s
+++ b/clang/test/Driver/relax.s
@@ -1,5 +1,5 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang -### -c -target x86_64-pc-linux -integrated-as -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck %s
+// RUN: %clang -### -c --target=x86_64-pc-linux -integrated-as -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck %s
 
 // CHECK: "-cc1as"
 // CHECK: "-mrelax-relocations=no"
diff --git a/clang/test/Driver/response-file-errs.c b/clang/test/Driver/response-file-errs.c
index efde7575a51e0..5331c8e308f48 100644
--- a/clang/test/Driver/response-file-errs.c
+++ b/clang/test/Driver/response-file-errs.c
@@ -11,5 +11,5 @@
 
 // If file in `@file` is a directory, it is an error.
 //
-// RUN: not %clang @%S/Inputs -### 2>&1 | FileCheck --check-prefix=DIRECTORY %s
-// DIRECTORY: cannot not open file '{{.*}}Inputs': {{[Ii]}}s a directory
+// RUN: not %clang @%S/Inputs -### 2>&1 | FileCheck -DMSG=%errc_EISDIR --check-prefix=DIRECTORY %s
+// DIRECTORY: cannot not open file '{{.*}}Inputs': [[MSG]]
diff --git a/clang/test/Driver/target-as.s b/clang/test/Driver/target-as.s
index 4881a330b56aa..2b639e95043c3 100644
--- a/clang/test/Driver/target-as.s
+++ b/clang/test/Driver/target-as.s
@@ -1,5 +1,5 @@
 // Make sure the -march is passed down to cc1as.
-// RUN: %clang -target i386-unknown-freebsd -### -c -integrated-as %s \
+// RUN: %clang --target=i386-unknown-freebsd -### -c -integrated-as %s \
 // RUN: -march=geode 2>&1 | FileCheck -check-prefix=TARGET %s
 //
 // TARGET: "-cc1as"
diff --git a/clang/test/Modules/string-literal-uniqueness.cpp b/clang/test/Modules/string-literal-uniqueness.cpp
new file mode 100644
index 0000000000000..34adc2b0303bd
--- /dev/null
+++ b/clang/test/Modules/string-literal-uniqueness.cpp
@@ -0,0 +1,60 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/a.cpp \
+// RUN:  -o %t/A.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/b.cpp \
+// RUN:  -fmodule-file=A=%t/A.pcm -o %t/B.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/c.cpp \
+// RUN:  -fmodule-file=A=%t/A.pcm -o %t/C.pcm
+
+// RUN: %clang_cc1 -std=c++20 -verify %t/main.cpp \
+// RUN:  -fmodule-file=A=%t/A.pcm \
+// RUN:  -fmodule-file=B=%t/B.pcm \
+// RUN:  -fmodule-file=C=%t/C.pcm
+
+// expected-no-diagnostics
+
+//--- a.cpp
+
+export module A;
+export consteval const char *hello() { return "hello"; }
+export constexpr const char *helloA0 = hello();
+export constexpr const char *helloA1 = helloA0;
+export constexpr const char *helloA2 = hello();
+
+//--- b.cpp
+
+export module B;
+import A;
+export constexpr const char *helloB1 = helloA0;
+export constexpr const char *helloB2 = hello();
+
+//--- c.cpp
+
+export module C;
+import A;
+export constexpr const char *helloC1 = helloA1;
+export constexpr const char *helloC2 = hello();
+
+//--- main.cpp
+
+import A;
+import B;
+import C;
+
+// These are valid: they refer to the same evaluation of the same constant.
+static_assert(helloA0 == helloA1);
+static_assert(helloA0 == helloB1);
+static_assert(helloA0 == helloC1);
+
+// These refer to distinct evaluations, and so may or may not be equal.
+static_assert(helloA1 == helloA2); // expected-error {{}} expected-note {{unspecified value}}
+static_assert(helloA1 == helloB2); // expected-error {{}} expected-note {{unspecified value}}
+static_assert(helloA1 == helloC2); // expected-error {{}} expected-note {{unspecified value}}
+static_assert(helloA2 == helloB2); // expected-error {{}} expected-note {{unspecified value}}
+static_assert(helloA2 == helloC2); // expected-error {{}} expected-note {{unspecified value}}
+static_assert(helloB2 == helloC2); // expected-error {{}} expected-note {{unspecified value}}
diff --git a/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
index 1c37d72de8614..b2d492d95945c 100644
--- a/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_contained_type_attr_error.hlsl
@@ -22,3 +22,7 @@ __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]]
 
 // expected-warning@+1{{attribute 'contained_type' is already applied with different arguments}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] [[hlsl::contained_type(int)]] h8;
+
+// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'contained_type' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+float [[hlsl::resource_class(UAV)]] [[hlsl::contained_type(float)]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
index 15685bd1a3baa..3b2c12e7a96c5 100644
--- a/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_is_rov_attr_error.hlsl
@@ -14,3 +14,7 @@ __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov(gibberish)]] res3
 
 // expected-warning@+1{{attribute 'is_rov' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] [[hlsl::is_rov]] res4;
+
+// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'is_rov' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+float [[hlsl::resource_class(UAV)]] [[hlsl::is_rov]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
index 83273426017ed..77530cbf9e4d9 100644
--- a/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_raw_buffer_attr_error.hlsl
@@ -11,3 +11,7 @@ __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer(gibberish)]]
 
 // expected-warning@+1{{attribute 'raw_buffer' is already applied}}
 __hlsl_resource_t [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] [[hlsl::raw_buffer]] res4;
+
+// expected-error@+2{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+// expected-error@+1{{attribute 'raw_buffer' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+float [[hlsl::resource_class(UAV)]] [[hlsl::raw_buffer]] res5;
diff --git a/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl b/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
index 01ff1c007e2b5..63e39daff949b 100644
--- a/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_class_attr_error.hlsl
@@ -17,3 +17,6 @@ __hlsl_resource_t [[hlsl::resource_class(SRV)]] [[hlsl::resource_class(SRV)]] e4
 
 // expected-error@+1{{'resource_class' attribute takes one argument}}
 __hlsl_resource_t [[hlsl::resource_class(SRV, "aa")]] e5;
+
+// expected-error@+1{{attribute 'resource_class' can be used only on HLSL intangible type '__hlsl_resource_t'}}
+float [[hlsl::resource_class(UAV)]] e6;
diff --git a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
index 301d61c0e906e..5e4ed96561a30 100644
--- a/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
+++ b/clang/test/ParserHLSL/hlsl_resource_handle_attrs.hlsl
@@ -3,7 +3,7 @@
 // CHECK: -ClassTemplateSpecializationDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> class RWBuffer definition implicit_instantiation
 // CHECK: -TemplateArgument type 'float'
 // CHECK: `-BuiltinType 0x{{[0-9a-f]+}} 'float'
-// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'float *
+// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'float *
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(float)]]
 // CHECK-SAME: ':'float *'
@@ -14,7 +14,7 @@ RWBuffer<float> Buffer1;
 // CHECK: -TemplateArgument type 'vector<float, 4>'
 // CHECK: `-ExtVectorType 0x{{[0-9a-f]+}} 'vector<float, 4>' 4
 // CHECK: `-BuiltinType 0x{{[0-9a-f]+}} 'float'
-// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit referenced h 'vector<float *, 4>
+// CHECK: -FieldDecl 0x{{[0-9a-f]+}} <<invalid sloc>> <invalid sloc> implicit h 'vector<float *, 4>
 // CHECK-SAME{LITERAL}: [[hlsl::resource_class(UAV)]
 // CHECK-SAME{LITERAL}: [[hlsl::is_rov]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(vector<float, 4>)]]
diff --git a/clang/test/Sema/attr-target-clones-aarch64.c b/clang/test/Sema/attr-target-clones-aarch64.c
index bfc08118a8ffc..6f0185816b0ac 100644
--- a/clang/test/Sema/attr-target-clones-aarch64.c
+++ b/clang/test/Sema/attr-target-clones-aarch64.c
@@ -24,17 +24,17 @@ int __attribute__((target_clones("rng", "fp16fml+fp", "default"))) redecl4(void)
 // expected-error@+3 {{'target_clones' attribute does not match previous declaration}}
 // expected-note@-2 {{previous declaration is here}}
 // expected-warning@+1 {{version list contains entries that don't impact code generation}}
-int __attribute__((target_clones("dgh+memtag+rpres", "ebf16+dpb+sha1", "default"))) redecl4(void) { return 1; }
+int __attribute__((target_clones("dgh+memtag+rpres", "ebf16+dpb", "default"))) redecl4(void) { return 1; }
 
 int __attribute__((target_version("flagm2"))) redef2(void) { return 1; }
 // expected-error@+2 {{multiversioned function redeclarations require identical target attributes}}
 // expected-note@-2 {{previous declaration is here}}
 int __attribute__((target_clones("flagm2", "default"))) redef2(void) { return 1; }
 
-int __attribute__((target_clones("f32mm", "f64mm", "sha1+fp"))) redef3(void) { return 1; }
+int __attribute__((target_clones("f32mm", "f64mm", "sha2+fp"))) redef3(void) { return 1; }
 // expected-error@+2 {{'target_clones' attribute does not match previous declaration}}
 // expected-note@-2 {{previous declaration is here}}
-int __attribute__((target_clones("f32mm", "sha1+fp", "f64mm"))) redef3(void) { return 1; }
+int __attribute__((target_clones("f32mm", "sha2+fp", "f64mm"))) redef3(void) { return 1; }
 
 int __attribute__((target_clones("rdm+lse+rdm", "lse+rdm"))) dup1(void) { return 1; }
 // expected-warning@+1 {{version list contains duplicate entries}}
diff --git a/clang/test/Sema/attr-target-version.c b/clang/test/Sema/attr-target-version.c
index 88a927a58f991..ed4f6c8556c99 100644
--- a/clang/test/Sema/attr-target-version.c
+++ b/clang/test/Sema/attr-target-version.c
@@ -36,7 +36,7 @@ void __attribute__((target_version("bti+flagm2"))) one(void) {}
 //expected-error@+1 {{multiversioned function redeclarations require identical target attributes}}
 void __attribute__((target_version("flagm2+bti"))) one(void) {}
 
-void __attribute__((target_version("ssbs+sha1"))) two(void) {}
+void __attribute__((target_version("ssbs+sha2"))) two(void) {}
 void __attribute__((target_version("ssbs+fp16fml"))) two(void) {}
 
 //expected-error@+1 {{'main' cannot be a multiversioned function}}
@@ -89,7 +89,7 @@ int bar() {
   return def();
 }
 // expected-error@+1 {{function declaration cannot become a multiversioned function after first usage}}
-int __attribute__((target_version("sha1"))) def(void) { return 1; }
+int __attribute__((target_version("sha2"))) def(void) { return 1; }
 
 int __attribute__((target_version("sve"))) prot();
 // expected-error@-1 {{multiversioned function must have a prototype}}
diff --git a/clang/test/Sema/diagnose_if.c b/clang/test/Sema/diagnose_if.c
index e9b8497d5ca4e..4df39916c031e 100644
--- a/clang/test/Sema/diagnose_if.c
+++ b/clang/test/Sema/diagnose_if.c
@@ -2,10 +2,10 @@
 
 #define _diagnose_if(...) __attribute__((diagnose_if(__VA_ARGS__)))
 
-void failure1(void) _diagnose_if(); // expected-error{{at least 3 arguments}}
-void failure2(void) _diagnose_if(0); // expected-error{{at least 3 arguments}}
-void failure3(void) _diagnose_if(0, ""); // expected-error{{at least 3 arguments}}
-void failure4(void) _diagnose_if(0, "", "error", 1); // expected-error{{expected string literal as argument}}
+void failure1(void) _diagnose_if(); // expected-error{{exactly 3 arguments}}
+void failure2(void) _diagnose_if(0); // expected-error{{exactly 3 arguments}}
+void failure3(void) _diagnose_if(0, ""); // expected-error{{exactly 3 arguments}}
+void failure4(void) _diagnose_if(0, "", "error", 1); // expected-error{{exactly 3 arguments}}
 void failure5(void) _diagnose_if(0, 0, "error"); // expected-error{{expected string literal as argument of 'diagnose_if' attribute}}
 void failure6(void) _diagnose_if(0, "", "invalid"); // expected-error{{invalid diagnostic type for 'diagnose_if'; use "error" or "warning" instead}}
 void failure7(void) _diagnose_if(0, "", "ERROR"); // expected-error{{invalid diagnostic type}}
diff --git a/clang/test/SemaCXX/builtins.cpp b/clang/test/SemaCXX/builtins.cpp
index f47ed3a1f7ebf..f99bb87b9cbd4 100644
--- a/clang/test/SemaCXX/builtins.cpp
+++ b/clang/test/SemaCXX/builtins.cpp
@@ -1,13 +1,21 @@
-// RUN: %clang_cc1 %s -fsyntax-only -verify -std=c++11 -fcxx-exceptions
-// RUN: %clang_cc1 %s -fsyntax-only -verify -std=c++1z -fcxx-exceptions
+// RUN: %clang_cc1 %s -fsyntax-only -verify -std=c++11 -fcxx-exceptions -fptrauth-intrinsics
+// RUN: %clang_cc1 %s -fsyntax-only -verify -std=c++1z -fcxx-exceptions -fptrauth-intrinsics
 typedef const struct __CFString * CFStringRef;
 #define CFSTR __builtin___CFStringMakeConstantString
+#define NSSTR __builtin___NSStringMakeConstantString
 
 void f() {
 #if !defined(__MVS__) && !defined(_AIX)
   // Builtin function __builtin___CFStringMakeConstantString is currently
   // unsupported on z/OS and AIX.
   (void)CFStringRef(CFSTR("Hello"));
+
+  constexpr bool a = CFSTR("Hello") == CFSTR("Hello");
+  // expected-error@-1 {{constant expression}}
+  // expected-note@-2 {{comparison against opaque constant address '&__builtin___CFStringMakeConstantString("Hello")'}}
+  constexpr bool b = NSSTR("Hello") == NSSTR("Hello");
+  // expected-error@-1 {{constant expression}}
+  // expected-note@-2 {{comparison against opaque constant address '&__builtin___NSStringMakeConstantString("Hello")'}}
 #endif
 }
 
@@ -47,7 +55,7 @@ void a(void) {}
 int n;
 void *p = __builtin_function_start(n);               // expected-error {{argument must be a function}}
 static_assert(__builtin_function_start(a) == a, ""); // expected-error {{static assertion expression is not an integral constant expression}}
-// expected-note@-1 {{comparison of addresses of literals has unspecified value}}
+// expected-note@-1 {{comparison against opaque constant address '&__builtin_function_start(a)'}}
 } // namespace function_start
 
 void no_ms_builtins() {
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 44ef540f41fa8..e2ea984b37cd0 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -std=c++20 -isystem %S/Inputs -fsyntax-only -verify=expected,cxx11_20,cxx20_23,pre-cxx23 -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
 // RUN: %clang_cc1 -std=c++11 -isystem %S/Inputs -fsyntax-only -verify=expected,cxx11_20,cxx11,pre-cxx23    -triple x86_64-linux -Wno-string-plus-int -Wno-pointer-arith -Wno-zero-length-array -Wno-c99-designator -fcxx-exceptions -pedantic %s -Wno-comment -Wno-tautological-pointer-compare -Wno-bool-conversion
 
+// This macro forces its argument to be constant-folded, even if it's not
+// otherwise a constant expression.
+#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
+
 namespace StaticAssertFoldTest {
 
 int x;
@@ -358,11 +362,36 @@ struct Str {
 
 extern char externalvar[];
 constexpr bool constaddress = (void *)externalvar == (void *)0x4000UL; // expected-error {{must be initialized by a constant expression}} expected-note {{reinterpret_cast}}
-constexpr bool litaddress = "foo" == "foo"; // expected-error {{must be initialized by a constant expression}}
-// expected-note@-1 {{comparison of addresses of literals has unspecified value}}
-// cxx20_23-warning@-2 {{comparison between two arrays is deprecated}}
 static_assert(0 != "foo", "");
 
+// OK: These string literals cannot possibly overlap.
+static_assert(+"foo" != +"bar", "");
+static_assert("xfoo" + 1 != "yfoo" + 1, "");
+static_assert(+"foot" != +"foo", "");
+static_assert(+"foo\0bar" != +"foo\0baz", "");
+
+// These can't overlap because the null terminator for UTF-16 is two bytes wide.
+static_assert(fold((const char*)u"A" != (const char*)"\0A\0x"), "");
+static_assert(fold((const char*)u"A" != (const char*)"A\0\0x"), "");
+
+constexpr const char *string = "hello";
+constexpr const char *also_string = string;
+static_assert(string == string, "");
+static_assert(string == also_string, "");
+
+// These strings may overlap, and so the result of the comparison is unknown.
+constexpr bool may_overlap_1 = +"foo" == +"foo"; // expected-error {{}} expected-note {{addresses of literals}}
+constexpr bool may_overlap_2 = +"foo" == +"foo\0bar"; // expected-error {{}} expected-note {{addresses of literals}}
+constexpr bool may_overlap_3 = +"foo" == "bar\0foo" + 4; // expected-error {{}} expected-note {{addresses of literals}}
+constexpr bool may_overlap_4 = "xfoo" + 1 == "xfoo" + 1; // expected-error {{}} expected-note {{addresses of literals}}
+
+// These may overlap even though they have different encodings.
+// One of these two comparisons is non-constant, but due to endianness we don't
+// know which one.
+constexpr bool may_overlap_different_encoding[] =
+  {fold((const char*)u"A" != (const char*)"xA\0\0\0x" + 1), fold((const char*)u"A" != (const char*)"x\0A\0\0x" + 1)};
+  // expected-error@-2 {{}} expected-note@-1 {{addresses of literals}}
+
 }
 
 namespace MaterializeTemporary {
@@ -1543,16 +1572,10 @@ namespace MutableMembers {
 
 namespace Fold {
 
-  // This macro forces its argument to be constant-folded, even if it's not
-  // otherwise a constant expression.
-  #define fold(x) (__builtin_constant_p(x) ? (x) : (x))
-
   constexpr int n = (long)(char*)123; // expected-error {{constant expression}} expected-note {{reinterpret_cast}}
   constexpr int m = fold((long)(char*)123); // ok
   static_assert(m == 123, "");
 
-  #undef fold
-
 }
 
 namespace DR1454 {
diff --git a/clang/test/SemaCXX/constant-expression-cxx14.cpp b/clang/test/SemaCXX/constant-expression-cxx14.cpp
index 70ab5dcd357c1..936d3600953b9 100644
--- a/clang/test/SemaCXX/constant-expression-cxx14.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx14.cpp
@@ -1306,3 +1306,18 @@ constexpr int field(int a) {
 static_assert(field(3), ""); // expected-error {{constant expression}} \
                              // expected-note {{in call to 'field(3)'}}
 }
+
+namespace literal_comparison {
+
+constexpr bool different_in_loop(bool b = false) {
+  if (b) return false;
+
+  const char *p[2] = {};
+  for (const char *&r : p)
+    r = "hello";
+  return p[0] == p[1]; // expected-note {{addresses of literals}}
+}
+constexpr bool check = different_in_loop();
+  // expected-error@-1 {{}} expected-note@-1 {{in call}}
+
+}
diff --git a/clang/test/SemaCXX/diagnose_if-warning-group.cpp b/clang/test/SemaCXX/diagnose_if-warning-group.cpp
deleted file mode 100644
index a39c0c0c33c9e..0000000000000
--- a/clang/test/SemaCXX/diagnose_if-warning-group.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-// RUN: %clang_cc1 %s -verify=expected,wall -fno-builtin -Wno-pedantic -Werror=comment -Wno-error=abi -Wfatal-errors=assume -Wno-fatal-errors=assume -Wno-format
-// RUN: %clang_cc1 %s -verify=expected,wno-all,pedantic,format -fno-builtin -Wno-all -Werror=comment -Wno-error=abi -Werror=assume -Wformat
-
-#define diagnose_if(...) __attribute__((diagnose_if(__VA_ARGS__)))
-
-#ifndef EMTY_WARNING_GROUP
-void bougus_warning() diagnose_if(true, "oh no", "warning", "bogus warning") {} // expected-error {{unknown warning group 'bogus warning'}}
-
-void show_in_system_header() diagnose_if(true, "oh no", "warning", "assume", "Banane") {} // expected-error {{'diagnose_if' attribute takes no more than 4 arguments}}
-#endif // EMTY_WARNING_GROUP
-
-template <bool b>
-void diagnose_if_wcomma() diagnose_if(b, "oh no", "warning", "comma") {}
-
-template <bool b>
-void diagnose_if_wcomment() diagnose_if(b, "oh no", "warning", "comment") {}
-
-void empty_warning_group() diagnose_if(true, "oh no", "warning", "") {} // expected-error {{unknown warning group ''}}
-void empty_warning_group_error() diagnose_if(true, "oh no", "error", "") {} // expected-error {{unknown warning group ''}}
-
-void diagnose_if_wabi_default_error() diagnose_if(true, "ABI stuff", "error", "abi") {}
-void diagnose_assume() diagnose_if(true, "Assume diagnostic", "warning", "assume") {}
-
-void Wall() diagnose_if(true, "oh no", "warning", "all") {}
-void Wpedantic() diagnose_if(true, "oh no", "warning", "pedantic") {}
-void Wformat_extra_args() diagnose_if(true, "oh no", "warning", "format-extra-args") {}
-
-void call() {
-  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
-  diagnose_if_wcomma<false>();
-  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
-  diagnose_if_wcomment<false>();
-
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wcomma"
-  diagnose_if_wcomma<true>();
-  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
-#pragma clang diagnostic pop
-
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wcomment"
-  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
-  diagnose_if_wcomment<true>();
-#pragma clang diagnostic pop
-
-  diagnose_if_wcomma<true>(); // expected-warning {{oh no}}
-  diagnose_if_wcomment<true>(); // expected-error {{oh no}}
-
-  diagnose_if_wabi_default_error(); // expected-warning {{ABI stuff}}
-  diagnose_assume(); // expected-error {{Assume diagnostic}}
-
-  // Make sure that the -Wassume diagnostic isn't fatal
-  diagnose_if_wabi_default_error(); // expected-warning {{ABI stuff}}
-
-  Wall(); // wall-warning {{oh no}}
-  Wpedantic(); // pedantic-warning {{oh no}}
-  Wformat_extra_args(); // format-warning {{oh no}}
-
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wformat"
-  Wformat_extra_args();
-#pragma clang diagnostic pop
-}
diff --git a/clang/test/SemaCXX/lambda-attributes.cpp b/clang/test/SemaCXX/lambda-attributes.cpp
new file mode 100644
index 0000000000000..97d23053b0f46
--- /dev/null
+++ b/clang/test/SemaCXX/lambda-attributes.cpp
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux -fsyntax-only -verify %s
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux -fsyntax-only -ast-dump %s | FileCheck %s
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-linux -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -std=c++23 -triple x86_64-unknown-linux -include-pch %t -ast-dump-all /dev/null | FileCheck %s
+// expected-no-diagnostics
+
+// Check that we both don't crash on transforming FunctionProtoType's
+// wrapped in type sugar and that we don't drop it when performing
+// instantiations either.
+
+#define PRESERVE __attribute__((preserve_most))
+
+// Skip to the instantiation of f().
+// CHECK: FunctionDecl {{.*}} f 'void ()' implicit_instantiation
+template <typename T>
+void f() {
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const __attribute__((preserve_most))':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+  (void) [] (T) __attribute__((preserve_most)) { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const {{\[}}[clang::annotate_type(...)]]':'void (int) const' implicit_instantiation
+  (void) [] (T) [[clang::annotate_type("foo")]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const {{\[}}[clang::annotate_type(...)]] {{\[}}[clang::annotate_type(...)]] {{\[}}[clang::annotate_type(...)]]':'void (int) const' implicit_instantiation
+  (void) [] (T) [[clang::annotate_type("foo")]]
+                [[clang::annotate_type("foo")]]
+                [[clang::annotate_type("foo")]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const __attribute__((preserve_most)) {{\[}}[clang::annotate_type(...)]]':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+  (void) [] (T) __attribute__((preserve_most))
+                [[clang::annotate_type("foo")]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const __attribute__((cdecl)) {{\[}}[clang::annotate_type(...)]]':'void (int) const' implicit_instantiation
+  (void) [] (T) __attribute__((cdecl))
+                [[clang::annotate_type("foo")]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const {{\[}}[clang::annotate_type(...)]]':'void (int) const' implicit_instantiation
+  (void) [] (T t) [[clang::annotate_type("foo", t)]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const __attribute__((preserve_most)) {{\[}}[clang::annotate_type(...)]]':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+  (void) [] (T t) __attribute__((preserve_most))
+                [[clang::annotate_type("foo", t, t, t, t)]] { };
+
+  // Check that the MacroQualifiedType is preserved.
+  // CHECK: CXXMethodDecl {{.*}} operator() 'PRESERVE void (int) __attribute__((preserve_most)) const':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+  (void) [] (T) PRESERVE { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'PRESERVE void (int) __attribute__((preserve_most)) const {{\[}}[clang::annotate_type(...)]]':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+  (void) [] (T) PRESERVE [[clang::annotate_type("foo")]] { };
+
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const {{\[}}[clang::annotate_type(...)]]':'void (int) const' implicit_instantiation
+  (void) [] (T) [[clang::annotate_type("foo")]] {
+    // CHECK: CXXMethodDecl {{.*}} operator() 'PRESERVE void (int) __attribute__((preserve_most)) const {{\[}}[clang::annotate_type(...)]]':'void (int) __attribute__((preserve_most)) const' implicit_instantiation
+    auto l = []<typename U = T> (U u = {}) PRESERVE [[clang::annotate_type("foo", u)]] { };
+
+    // CHECK: DeclRefExpr {{.*}} 'PRESERVE void (int) __attribute__((preserve_most)) const {{\[}}[clang::annotate_type(...)]]':'void (int) __attribute__((preserve_most)) const' lvalue CXXMethod
+    l();
+  };
+}
+
+void g() {
+  f<int>();
+}
diff --git a/clang/test/SemaCXX/lambda-conversion-op-cc.cpp b/clang/test/SemaCXX/lambda-conversion-op-cc.cpp
index 3632f8c8c80aa..6d9f5d702e132 100644
--- a/clang/test/SemaCXX/lambda-conversion-op-cc.cpp
+++ b/clang/test/SemaCXX/lambda-conversion-op-cc.cpp
@@ -44,19 +44,19 @@ void useage() {
 
   // CHECK: VarDecl {{.*}} vectorcall '
   // CHECK: LambdaExpr
-  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int, float, double) __attribute__((vectorcall)) const'
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int, float, double) const __attribute__((vectorcall))':'void (int, float, double) __attribute__((vectorcall)) const'
   // CHECK: CXXConversionDecl {{.*}} operator void (*)(int, float, double) __attribute__((vectorcall)) 'void
   // CHECK: CXXMethodDecl {{.*}} __invoke 'void (int, float, double) __attribute__((vectorcall))' static inline
 
   // WIN32: VarDecl {{.*}} thiscall '
   // WIN32: LambdaExpr
-  // WIN32: CXXMethodDecl {{.*}} operator() 'void (int, float, double) __attribute__((thiscall)) const'
+  // WIN32: CXXMethodDecl {{.*}} operator() 'void (int, float, double) const __attribute__((thiscall))':'void (int, float, double) __attribute__((thiscall)) const'
   // WIN32: CXXConversionDecl {{.*}} operator void (*)(int, float, double) 'void
   // WIN32: CXXMethodDecl {{.*}} __invoke 'void (int, float, double)' static inline
 
   // CHECK: VarDecl {{.*}} cdecl '
   // CHECK: LambdaExpr
-  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int, float, double) const'
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int, float, double) const __attribute__((cdecl))':'void (int, float, double) const'
   // NODEF: CXXConversionDecl {{.*}} operator void (*)(int, float, double) 'void
   // NODEF: CXXMethodDecl {{.*}} __invoke 'void (int, float, double)' static inline
   // VECTDEF: CXXConversionDecl {{.*}} operator void (*)(int, float, double) __attribute__((vectorcall)) 'void
@@ -108,8 +108,8 @@ void useage() {
   // CHECK: LambdaExpr
   // CHECK: FunctionTemplateDecl {{.*}} operator()
   // CHECK: CXXMethodDecl {{.*}} operator() 'auto (auto) __attribute__((vectorcall)) const' inline
-  // CHECK: CXXMethodDecl {{.*}} operator() 'void (char) __attribute__((vectorcall)) const' implicit_instantiation inline
-  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) __attribute__((vectorcall)) const' implicit_instantiation inline
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (char) const __attribute__((vectorcall))':'void (char) __attribute__((vectorcall)) const' implicit_instantiation inline
+  // CHECK: CXXMethodDecl {{.*}} operator() 'void (int) const __attribute__((vectorcall))':'void (int) __attribute__((vectorcall)) const' implicit_instantiation inline
   // CHECK: FunctionTemplateDecl {{.*}} operator auto (*)(type-parameter-0-0) __attribute__((vectorcall))
   // LIN64: CXXConversionDecl {{.*}} operator auto (*)(type-parameter-0-0) __attribute__((vectorcall)) 'auto (*() const noexcept)(auto) __attribute__((vectorcall))'
   // LIN64: CXXConversionDecl {{.*}} operator auto (*)(char) __attribute__((vectorcall)) 'void (*() const noexcept)(char) __attribute__((vectorcall))'
diff --git a/clang/test/SemaCXX/ptrauth-sign-constant.cpp b/clang/test/SemaCXX/ptrauth-sign-constant.cpp
new file mode 100644
index 0000000000000..396962e33e2fa
--- /dev/null
+++ b/clang/test/SemaCXX/ptrauth-sign-constant.cpp
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -triple arm64-apple-ios -std=c++17 -Wno-vla -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c++17 -Wno-vla -fsyntax-only -verify -fptrauth-intrinsics %s
+
+int n;
+constexpr bool compare_result = __builtin_ptrauth_sign_constant(&n, 2, 0) == &n;
+// expected-error@-1 {{constant expression}}
+// expected-note@-2 {{comparison against opaque constant address '&__builtin_ptrauth_sign_constant(&n, 2, 0)'}}
\ No newline at end of file
diff --git a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
index 871fe5e4553cc..b4b376fe0d114 100644
--- a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
+++ b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
@@ -236,9 +236,8 @@ void printCommands(ArrayRef<StringRef> CmdArgs) {
   if (CmdArgs.empty())
     return;
 
-  llvm::errs() << " \"" << CmdArgs.front() << "\" ";
-  llvm::errs() << llvm::join(std::next(CmdArgs.begin()), CmdArgs.end(), " ")
-               << "\n";
+  errs() << " \"" << CmdArgs.front() << "\" ";
+  errs() << join(std::next(CmdArgs.begin()), CmdArgs.end(), " ") << "\n";
 }
 
 /// A minimum symbol interface that provides the necessary information to
@@ -329,12 +328,12 @@ Expected<std::unique_ptr<lto::LTO>> createLTO(const ArgList &Args) {
   lto::ThinBackend Backend;
   unsigned Jobs = 0;
   if (auto *Arg = Args.getLastArg(OPT_jobs))
-    if (!llvm::to_integer(Arg->getValue(), Jobs) || Jobs == 0)
+    if (!to_integer(Arg->getValue(), Jobs) || Jobs == 0)
       reportError(createStringError("%s: expected a positive integer, got '%s'",
                                     Arg->getSpelling().data(),
                                     Arg->getValue()));
-  Backend = lto::createInProcessThinBackend(
-      llvm::heavyweight_hardware_concurrency(Jobs));
+  Backend =
+      lto::createInProcessThinBackend(heavyweight_hardware_concurrency(Jobs));
 
   Conf.CPU = Args.getLastArgValue(OPT_arch);
   Conf.Options = codegen::InitTargetOptionsFromCodeGenFlags(Triple);
@@ -378,7 +377,7 @@ Expected<std::unique_ptr<lto::LTO>> createLTO(const ArgList &Args) {
 
   unsigned Partitions = 1;
   if (auto *Arg = Args.getLastArg(OPT_lto_partitions))
-    if (!llvm::to_integer(Arg->getValue(), Partitions) || Partitions == 0)
+    if (!to_integer(Arg->getValue(), Partitions) || Partitions == 0)
       reportError(createStringError("%s: expected a positive integer, got '%s'",
                                     Arg->getSpelling().data(),
                                     Arg->getValue()));
@@ -510,7 +509,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
       InputFiles.emplace_back(std::move(*BufferOrErr), /*IsLazy=*/false);
       break;
     case file_magic::archive: {
-      Expected<std::unique_ptr<llvm::object::Archive>> LibFile =
+      Expected<std::unique_ptr<object::Archive>> LibFile =
           object::Archive::create(Buffer);
       if (!LibFile)
         return LibFile.takeError();
@@ -563,7 +562,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
   for (auto &Input : LinkerInput)
     if (identify_magic(Input->getBuffer()) == file_magic::bitcode)
       BitcodeFiles.emplace_back(std::move(Input));
-  llvm::erase_if(LinkerInput, [](const auto &F) { return !F; });
+  erase_if(LinkerInput, [](const auto &F) { return !F; });
 
   // Run the LTO pipeline on the extracted inputs.
   SmallVector<StringRef> Files;
@@ -574,7 +573,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
     lto::LTO &LTOBackend = **LTOBackendOrErr;
     for (auto &BitcodeFile : BitcodeFiles) {
       Expected<std::unique_ptr<lto::InputFile>> BitcodeFileOrErr =
-          llvm::lto::InputFile::create(*BitcodeFile);
+          lto::InputFile::create(*BitcodeFile);
       if (!BitcodeFileOrErr)
         return BitcodeFileOrErr.takeError();
 
@@ -638,7 +637,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
       if (std::error_code EC = sys::fs::openFileForWrite(TempFile, FD))
         reportError(errorCodeToError(EC));
       return std::make_unique<CachedFileStream>(
-          std::make_unique<llvm::raw_fd_ostream>(FD, true));
+          std::make_unique<raw_fd_ostream>(FD, true));
     };
 
     if (Error Err = LTOBackend.run(AddStream))
@@ -655,8 +654,10 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
     }
   }
 
-  // Copy all of the input files to a new file ending in `.cubin`. The 'nvlink'
+  // Create a copy for each file to a new file ending in `.cubin`. The 'nvlink'
   // linker requires all NVPTX inputs to have this extension for some reason.
+  // We don't use a symbolic link because it's not supported on Windows and some
+  // of this input files could be extracted from an archive.
   for (auto &Input : LinkerInput) {
     auto TempFileOrErr = createTempFile(
         Args, sys::path::stem(Input->getBufferIdentifier()), "cubin");
@@ -667,7 +668,7 @@ Expected<SmallVector<StringRef>> getInput(const ArgList &Args) {
     if (!OutputOrErr)
       return OutputOrErr.takeError();
     std::unique_ptr<FileOutputBuffer> Output = std::move(*OutputOrErr);
-    llvm::copy(Input->getBuffer(), Output->getBufferStart());
+    copy(Input->getBuffer(), Output->getBufferStart());
     if (Error E = Output->commit())
       return E;
     Files.emplace_back(Args.MakeArgString(*TempFileOrErr));
@@ -704,8 +705,8 @@ Error runNVLink(ArrayRef<StringRef> Files, const ArgList &Args) {
     Arg->render(Args, NewLinkerArgs);
   }
 
-  llvm::transform(Files, std::back_inserter(NewLinkerArgs),
-                  [&](StringRef Arg) { return Args.MakeArgString(Arg); });
+  transform(Files, std::back_inserter(NewLinkerArgs),
+            [&](StringRef Arg) { return Args.MakeArgString(Arg); });
 
   SmallVector<StringRef> LinkerArgs({*NVLinkPath});
   if (!Args.hasArg(OPT_o))
diff --git a/clang/tools/diagtool/ListWarnings.cpp b/clang/tools/diagtool/ListWarnings.cpp
index 9f9647126dd8a..a71f6e3a66c8e 100644
--- a/clang/tools/diagtool/ListWarnings.cpp
+++ b/clang/tools/diagtool/ListWarnings.cpp
@@ -53,13 +53,13 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
     const unsigned diagID = DR.DiagID;
 
-    if (DiagnosticIDs{}.isNote(diagID))
+    if (DiagnosticIDs::isBuiltinNote(diagID))
       continue;
 
-    if (!DiagnosticIDs{}.isWarningOrExtension(diagID))
+    if (!DiagnosticIDs::isBuiltinWarningOrExtension(diagID))
       continue;
 
-    Entry entry(DR.getName(), DiagnosticIDs{}.getWarningOptionForDiag(diagID));
+    Entry entry(DR.getName(), DiagnosticIDs::getWarningOptionForDiag(diagID));
 
     if (entry.Flag.empty())
       Unflagged.push_back(entry);
@@ -97,3 +97,4 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
 
   return 0;
 }
+
diff --git a/clang/tools/diagtool/ShowEnabledWarnings.cpp b/clang/tools/diagtool/ShowEnabledWarnings.cpp
index caf67223921d4..66a295db054c3 100644
--- a/clang/tools/diagtool/ShowEnabledWarnings.cpp
+++ b/clang/tools/diagtool/ShowEnabledWarnings.cpp
@@ -117,10 +117,10 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
   for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
     unsigned DiagID = DR.DiagID;
 
-    if (DiagnosticIDs{}.isNote(DiagID))
+    if (DiagnosticIDs::isBuiltinNote(DiagID))
       continue;
 
-    if (!DiagnosticIDs{}.isWarningOrExtension(DiagID))
+    if (!DiagnosticIDs::isBuiltinWarningOrExtension(DiagID))
       continue;
 
     DiagnosticsEngine::Level DiagLevel =
@@ -128,7 +128,7 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
     if (DiagLevel == DiagnosticsEngine::Ignored)
       continue;
 
-    StringRef WarningOpt = DiagnosticIDs{}.getWarningOptionForDiag(DiagID);
+    StringRef WarningOpt = DiagnosticIDs::getWarningOptionForDiag(DiagID);
     Active.push_back(PrettyDiag(DR.getName(), WarningOpt, DiagLevel));
   }
 
diff --git a/clang/tools/libclang/CXStoredDiagnostic.cpp b/clang/tools/libclang/CXStoredDiagnostic.cpp
index 6fb3050f5f844..03018229549bd 100644
--- a/clang/tools/libclang/CXStoredDiagnostic.cpp
+++ b/clang/tools/libclang/CXStoredDiagnostic.cpp
@@ -51,9 +51,7 @@ CXString CXStoredDiagnostic::getSpelling() const {
 
 CXString CXStoredDiagnostic::getDiagnosticOption(CXString *Disable) const {
   unsigned ID = Diag.getID();
-  if (DiagnosticIDs::IsCustomDiag(ID))
-    return cxstring::createEmpty();
-  StringRef Option = DiagnosticIDs{}.getWarningOptionForDiag(ID);
+  StringRef Option = DiagnosticIDs::getWarningOptionForDiag(ID);
   if (!Option.empty()) {
     if (Disable)
       *Disable = cxstring::createDup((Twine("-Wno-") + Option).str());
diff --git a/clang/unittests/AST/MatchVerifier.h b/clang/unittests/AST/MatchVerifier.h
index 60bb4a8716ae8..e28946977de88 100644
--- a/clang/unittests/AST/MatchVerifier.h
+++ b/clang/unittests/AST/MatchVerifier.h
@@ -88,38 +88,13 @@ MatchVerifier<NodeType>::match(const std::string &Code,
 
   StringRef FileName;
   switch (L) {
-  case Lang_C89:
-    Args.push_back("-std=c89");
-    FileName = "input.c";
-    break;
-  case Lang_C99:
-    Args.push_back("-std=c99");
-    FileName = "input.c";
-    break;
-  case Lang_CXX03:
-    Args.push_back("-std=c++03");
-    FileName = "input.cc";
-    break;
-  case Lang_CXX11:
-    Args.push_back("-std=c++11");
-    FileName = "input.cc";
-    break;
-  case Lang_CXX14:
-    Args.push_back("-std=c++14");
-    FileName = "input.cc";
-    break;
-  case Lang_CXX17:
-    Args.push_back("-std=c++17");
-    FileName = "input.cc";
-    break;
-  case Lang_CXX20:
-    Args.push_back("-std=c++20");
-    FileName = "input.cc";
-    break;
-  case Lang_CXX23:
-    Args.push_back("-std=c++23");
-    FileName = "input.cc";
+#define TESTLANGUAGE(lang, version, std_flag, version_index)                   \
+  case Lang_##lang##version:                                                   \
+    Args.push_back("-std=" #std_flag);                                         \
+    FileName = getFilenameForTesting(Lang_##lang##version);                    \
     break;
+#include "clang/Testing/TestLanguage.def"
+
   case Lang_OpenCL:
     Args.push_back("-cl-no-stdinc");
     FileName = "input.cl";
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index f2eaf19d61402..ebf548eb25431 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -1224,7 +1224,7 @@ TEST_P(ASTMatchersTest, CastExpression_MatchesImplicitCasts) {
 }
 
 TEST_P(ASTMatchersTest, CastExpr_DoesNotMatchNonCasts) {
-  if (GetParam().Language == Lang_C89 || GetParam().Language == Lang_C99) {
+  if (GetParam().isC()) {
     // This does have a cast in C
     EXPECT_TRUE(matches("char c = '0';", implicitCastExpr()));
   } else {
@@ -1678,7 +1678,7 @@ TEST_P(ASTMatchersTest, FunctionProtoType) {
 }
 
 TEST_P(ASTMatchersTest, FunctionProtoType_C) {
-  if (!GetParam().isC()) {
+  if (!GetParam().isCOrEarlier(17)) {
     return;
   }
   EXPECT_TRUE(notMatches("void f();", functionProtoType()));
@@ -2030,8 +2030,6 @@ TEST_P(ASTMatchersTest,
 template <typename T>
 class VerifyAncestorHasChildIsEqual : public BoundNodesCallback {
 public:
-  bool run(const BoundNodes *Nodes) override { return false; }
-
   bool run(const BoundNodes *Nodes, ASTContext *Context) override {
     const T *Node = Nodes->getNodeAs<T>("");
     return verify(*Nodes, *Context, Node);
@@ -2745,8 +2743,11 @@ TEST(MatchFinderAPI, MatchesDynamic) {
 
 static std::vector<TestClangConfig> allTestClangConfigs() {
   std::vector<TestClangConfig> all_configs;
-  for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
-                            Lang_CXX14, Lang_CXX17, Lang_CXX20, Lang_CXX23}) {
+  for (TestLanguage lang : {
+#define TESTLANGUAGE(lang, version, std_flag, version_index)                   \
+  Lang_##lang##version,
+#include "clang/Testing/TestLanguage.def"
+       }) {
     TestClangConfig config;
     config.Language = lang;
 
@@ -2770,8 +2771,11 @@ static std::vector<TestClangConfig> allTestClangConfigs() {
   return all_configs;
 }
 
-INSTANTIATE_TEST_SUITE_P(ASTMatchersTests, ASTMatchersTest,
-                         testing::ValuesIn(allTestClangConfigs()));
+INSTANTIATE_TEST_SUITE_P(
+    ASTMatchersTests, ASTMatchersTest, testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 } // namespace ast_matchers
 } // namespace clang
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTest.h b/clang/unittests/ASTMatchers/ASTMatchersTest.h
index e981299531574..2327b0225ae0e 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTest.h
+++ b/clang/unittests/ASTMatchers/ASTMatchersTest.h
@@ -9,12 +9,46 @@
 #ifndef LLVM_CLANG_UNITTESTS_ASTMATCHERS_ASTMATCHERSTEST_H
 #define LLVM_CLANG_UNITTESTS_ASTMATCHERS_ASTMATCHERSTEST_H
 
+#include "clang/AST/APValue.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Frontend/ASTUnit.h"
+#include "clang/Serialization/PCHContainerOperations.h"
 #include "clang/Testing/CommandLineArgs.h"
 #include "clang/Testing/TestClangConfig.h"
+#include "clang/Tooling/ArgumentsAdjusters.h"
+#include "clang/Tooling/FixIt.h"
 #include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
+#include <cstddef>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
 
 namespace clang {
 namespace ast_matchers {
@@ -28,7 +62,6 @@ using clang::tooling::runToolOnCodeWithArgs;
 class BoundNodesCallback {
 public:
   virtual ~BoundNodesCallback() {}
-  virtual bool run(const BoundNodes *BoundNodes) = 0;
   virtual bool run(const BoundNodes *BoundNodes, ASTContext *Context) = 0;
   virtual void onEndOfTranslationUnit() {}
 };
@@ -294,7 +327,8 @@ template <typename T>
 testing::AssertionResult matchAndVerifyResultConditionally(
     const Twine &Code, const T &AMatcher,
     std::unique_ptr<BoundNodesCallback> FindResultVerifier, bool ExpectResult,
-    ArrayRef<std::string> Args = {}, StringRef Filename = "input.cc") {
+    ArrayRef<std::string> Args = {}, StringRef Filename = "input.cc",
+    const FileContentMappings &VirtualMappedFiles = {}) {
   bool VerifiedResult = false;
   MatchFinder Finder;
   VerifyMatch VerifyVerifiedResult(std::move(FindResultVerifier),
@@ -311,7 +345,9 @@ testing::AssertionResult matchAndVerifyResultConditionally(
   // choices that we made above.
   llvm::copy(Args, std::back_inserter(CompileArgs));
 
-  if (!runToolOnCodeWithArgs(Factory->create(), Code, CompileArgs, Filename)) {
+  if (!runToolOnCodeWithArgs(
+          Factory->create(), Code, CompileArgs, Filename, "clang-tool",
+          std::make_shared<PCHContainerOperations>(), VirtualMappedFiles)) {
     return testing::AssertionFailure() << "Parsing error in \"" << Code << "\"";
   }
   if (!VerifiedResult && ExpectResult) {
@@ -325,7 +361,9 @@ testing::AssertionResult matchAndVerifyResultConditionally(
   VerifiedResult = false;
   SmallString<256> Buffer;
   std::unique_ptr<ASTUnit> AST(buildASTFromCodeWithArgs(
-      Code.toStringRef(Buffer), CompileArgs, Filename));
+      Code.toStringRef(Buffer), CompileArgs, Filename, "clang-tool",
+      std::make_shared<PCHContainerOperations>(),
+      tooling::getClangStripDependencyFileAdjuster(), VirtualMappedFiles));
   if (!AST.get())
     return testing::AssertionFailure()
            << "Parsing error in \"" << Code << "\" while building AST";
@@ -403,7 +441,7 @@ template <typename T> class VerifyIdIsBoundTo : public BoundNodesCallback {
     EXPECT_EQ("", Name);
   }
 
-  bool run(const BoundNodes *Nodes) override {
+  bool run(const BoundNodes *Nodes, ASTContext * /*Context*/) override {
     const BoundNodes::IDToNodeMap &M = Nodes->getMap();
     if (Nodes->getNodeAs<T>(Id)) {
       ++Count;
@@ -426,10 +464,6 @@ template <typename T> class VerifyIdIsBoundTo : public BoundNodesCallback {
     return false;
   }
 
-  bool run(const BoundNodes *Nodes, ASTContext *Context) override {
-    return run(Nodes);
-  }
-
 private:
   const std::string Id;
   const int ExpectedCount;
@@ -438,6 +472,355 @@ template <typename T> class VerifyIdIsBoundTo : public BoundNodesCallback {
   std::string Name;
 };
 
+namespace detail {
+template <typename T>
+using hasDump_t = decltype(std::declval<const T &>().dump());
+template <typename T>
+constexpr bool hasDump = llvm::is_detected<hasDump_t, T>::value;
+
+template <typename T>
+using hasGetSourceRange_t =
+    decltype(std::declval<const T &>().getSourceRange());
+template <typename T>
+constexpr bool hasGetSourceRange =
+    llvm::is_detected<hasGetSourceRange_t, T>::value;
+
+template <typename T, std::enable_if_t<hasGetSourceRange<T>, bool> = true>
+std::optional<std::string> getText(const T *const Node,
+                                   const ASTContext &Context) {
+  return tooling::fixit::getText(*Node, Context).str();
+}
+inline std::optional<std::string> getText(const Attr *const Attribute,
+                                          const ASTContext &) {
+  return Attribute->getSpelling();
+}
+inline std::optional<std::string> getText(const void *const,
+                                          const ASTContext &) {
+  return std::nullopt;
+}
+
+template <typename T>
+auto getSourceRange(const T *const Node)
+    -> std::optional<decltype(Node->getSourceRange())> {
+  return Node->getSourceRange();
+}
+inline std::optional<SourceRange> getSourceRange(const void *const) {
+  return std::nullopt;
+}
+
+template <typename T>
+auto getLocation(const T *const Node)
+    -> std::optional<decltype(Node->getLocation())> {
+  return Node->getLocation();
+}
+inline std::optional<SourceLocation> getLocation(const void *const) {
+  return std::nullopt;
+}
+
+template <typename T>
+auto getBeginLoc(const T *const Node)
+    -> std::optional<decltype(Node->getBeginLoc())> {
+  return Node->getBeginLoc();
+}
+inline std::optional<SourceLocation> getBeginLoc(const void *const) {
+  return std::nullopt;
+}
+
+inline std::optional<SourceLocation>
+getLocOfTagDeclFromType(const Type *const Node) {
+  if (Node->isArrayType())
+    if (const auto *const AType = Node->getPointeeOrArrayElementType()) {
+      return getLocOfTagDeclFromType(AType);
+    }
+  if (const auto *const TDecl = Node->getAsTagDecl()) {
+    return TDecl->getLocation();
+  }
+  return std::nullopt;
+}
+inline std::optional<SourceLocation>
+getLocOfTagDeclFromType(const void *const Node) {
+  return std::nullopt;
+}
+
+template <typename T>
+auto getExprLoc(const T *const Node)
+    -> std::optional<decltype(Node->getBeginLoc())> {
+  return Node->getBeginLoc();
+}
+inline std::optional<SourceLocation> getExprLoc(const void *const) {
+  return std::nullopt;
+}
+
+// Provides a string for test failures to show what was matched.
+template <typename T>
+static std::optional<std::string>
+getNodeDescription(const T *const Node, const ASTContext *const Context) {
+  if constexpr (std::is_same_v<T, QualType>) {
+    return Node->getAsString();
+  }
+  if constexpr (std::is_base_of_v<NamedDecl, T>) {
+    return Node->getNameAsString();
+  }
+  if constexpr (std::is_base_of_v<Type, T>) {
+    return QualType{Node, 0}.getAsString();
+  }
+
+  return detail::getText(Node, *Context);
+}
+
+template <typename T>
+bool shouldIgnoreNode(const T *const Node, const ASTContext &Context) {
+  if (const auto Range = detail::getSourceRange(Node); Range.has_value()) {
+    if (Range->isInvalid() ||
+        !Context.getSourceManager().isInMainFile(Range->getBegin()))
+      return true;
+  } else if (const auto Loc = detail::getExprLoc(Node); Loc.has_value()) {
+    if (Loc->isInvalid() || !Context.getSourceManager().isInMainFile(*Loc))
+      return true;
+  } else if (const auto Loc = detail::getLocation(Node); Loc.has_value()) {
+    if (Loc->isInvalid() || !Context.getSourceManager().isInMainFile(*Loc))
+      return true;
+  } else if (const auto Loc = detail::getBeginLoc(Node); Loc.has_value()) {
+    if (Loc->isInvalid() || !Context.getSourceManager().isInMainFile(*Loc))
+      return true;
+  } else if (const auto Loc = detail::getLocOfTagDeclFromType(Node);
+             Loc.has_value()) {
+    if (Loc->isInvalid() || !Context.getSourceManager().isInMainFile(*Loc))
+      return true;
+  }
+  return false;
+}
+} // namespace detail
+
+enum class MatchKind {
+  Code,
+  Name,
+  TypeStr,
+};
+
+inline llvm::StringRef toString(const MatchKind Kind) {
+  switch (Kind) {
+  case MatchKind::Code:
+    return "Code";
+  case MatchKind::Name:
+    return "Name";
+  case MatchKind::TypeStr:
+    return "TypeStr";
+  }
+  llvm_unreachable("Unhandled MatchKind");
+}
+
+template <typename T> class VerifyBoundNodeMatch : public BoundNodesCallback {
+public:
+  class Match {
+  public:
+    Match(const MatchKind Kind, std::string MatchString,
+          const size_t MatchCount = 1)
+        : Kind(Kind), MatchString(std::move(MatchString)),
+          RemainingMatches(MatchCount) {}
+
+    bool shouldRemoveMatched(const T *const Node) {
+      --RemainingMatches;
+      return RemainingMatches == 0U;
+    }
+
+    template <typename U>
+    static std::optional<std::string>
+    getMatchText(const U *const Node, const ASTContext &Context,
+                 const MatchKind Kind, const bool EmitFailures = true) {
+      if constexpr (std::is_same_v<U, NestedNameSpecifier>) {
+        if (const IdentifierInfo *const Info = Node->getAsIdentifier())
+          return Info->getName().str();
+        if (const NamespaceDecl *const NS = Node->getAsNamespace())
+          return getMatchText(NS, Context, Kind, EmitFailures);
+        if (const NamespaceAliasDecl *const Alias = Node->getAsNamespaceAlias())
+          return getMatchText(Alias, Context, Kind, EmitFailures);
+        if (const CXXRecordDecl *const RDecl = Node->getAsRecordDecl())
+          return getMatchText(RDecl, Context, Kind, EmitFailures);
+        if (const Type *const RDecl = Node->getAsType())
+          return getMatchText(RDecl, Context, Kind, EmitFailures);
+      }
+
+      switch (Kind) {
+      case MatchKind::Code:
+        return detail::getText(Node, Context);
+      case MatchKind::Name:
+        return getNameText(Node, EmitFailures);
+      case MatchKind::TypeStr:
+        return getTypeStrText(Node, EmitFailures);
+      }
+    }
+
+    bool isMatch(const T *const Node, const ASTContext &Context) const {
+      if (const auto OptMatchText = getMatchText(Node, Context, Kind))
+        return *OptMatchText == MatchString;
+
+      return false;
+    }
+
+    std::string getAsString() const {
+      return llvm::formatv("MatchKind: {0}, MatchString: '{1}', "
+                           "RemainingMatches: {2}",
+                           toString(Kind), MatchString, RemainingMatches)
+          .str();
+    }
+
+    MatchKind getMatchKind() const { return Kind; }
+    llvm::StringRef getMatchString() const { return MatchString; }
+
+  private:
+    template <typename U>
+    static std::optional<std::string>
+    getNameText(const U *const Node, const bool EmitFailures = true) {
+      if constexpr (std::is_base_of_v<Decl, U>) {
+        if (const auto *const NDecl = llvm::dyn_cast<NamedDecl>(Node))
+          return NDecl->getNameAsString();
+        if (EmitFailures)
+          ADD_FAILURE() << "'MatchKind::Name' requires 'U' to be a'NamedDecl'.";
+      }
+
+      if (EmitFailures)
+        ADD_FAILURE() << "'MatchKind::Name' requires 'U' to be a "
+                         "'NamedDecl', but 'U' is not derived from 'Decl'.";
+      return std::nullopt;
+    }
+
+    template <typename U>
+    static std::optional<std::string>
+    getTypeStrText(const U *const Node, const bool EmitFailures = true) {
+      if constexpr (std::is_base_of_v<Type, U>)
+        return QualType(Node, 0).getAsString();
+      if constexpr (std::is_base_of_v<Decl, U>) {
+        if (const auto *const TDecl = llvm::dyn_cast<TypeDecl>(Node))
+          return getTypeStrText(TDecl->getTypeForDecl());
+        if (const auto *const VDecl = llvm::dyn_cast<ValueDecl>(Node))
+          return VDecl->getType().getAsString();
+      }
+      if (EmitFailures)
+        ADD_FAILURE() << "Match kind is 'TypeStr', but node of type 'U' is "
+                         "not handled.";
+      return std::nullopt;
+    }
+
+    MatchKind Kind;
+    std::string MatchString;
+    size_t RemainingMatches;
+  };
+
+  VerifyBoundNodeMatch(std::string Id, std::vector<Match> Matches)
+      : Id(std::move(Id)), ExpectedMatches(std::move(Matches)),
+        Matches(ExpectedMatches) {}
+
+  bool run(const BoundNodes *const Nodes, ASTContext *const Context) override {
+    const auto *const Node = Nodes->getNodeAs<T>(Id);
+    if (Node == nullptr) {
+      ADD_FAILURE() << "Expected Id '" << Id << "' to be bound to 'T'.";
+      return true;
+    }
+
+    if constexpr (std::is_base_of_v<Decl, T>)
+      if (const auto *const NDecl = llvm::dyn_cast<NamedDecl>(Node))
+        if (const auto *Identifier = NDecl->getIdentifier();
+            Identifier != nullptr && Identifier->getBuiltinID() > 0)
+          return true;
+
+    if (detail::shouldIgnoreNode(Node, *Context)) {
+      return false;
+    }
+
+    const auto Iter = llvm::find_if(Matches, [Node, Context](const Match &M) {
+      return M.isMatch(Node, *Context);
+    });
+    if (Iter == Matches.end()) {
+      const auto NodeText =
+          detail::getNodeDescription(Node, Context).value_or("<unknown>");
+      const auto IsMultilineNodeText = NodeText.find('\n') != std::string::npos;
+      ADD_FAILURE() << "No match of node '" << (IsMultilineNodeText ? "\n" : "")
+                    << NodeText << (IsMultilineNodeText ? "\n" : "")
+                    << "' was expected.\n"
+                    << "No match with remaining matches:"
+                    << getMatchComparisonText(Matches, Node, *Context)
+                    << "Match strings of Node for possible intended matches:"
+                    << getPossibleMatchStrings(Node, *Context)
+                    << "Already found matches:"
+                    << getMatchesAsString(FoundMatches) << "Expected matches:"
+                    << getMatchesAsString(ExpectedMatches);
+      if constexpr (detail::hasDump<T>)
+        Node->dump();
+      return true;
+    }
+
+    if (Iter->shouldRemoveMatched(Node)) {
+      FoundMatches.push_back(*Iter);
+      Matches.erase(Iter);
+    }
+
+    return true;
+  }
+
+  void onEndOfTranslationUnit() override {
+    if (!ExpectedMatches.empty() && Matches.size() == ExpectedMatches.size())
+      ADD_FAILURE() << "No matches were found.\n"
+                    << "Expected matches:"
+                    << getMatchesAsString(ExpectedMatches);
+    else
+      EXPECT_TRUE(Matches.empty())
+          << "Not all expected matches were found.\n"
+          << "Remaining matches:" << getMatchesAsString(Matches)
+          << "Already found matches:" << getMatchesAsString(FoundMatches)
+          << "Expected matches:" << getMatchesAsString(ExpectedMatches);
+
+    Matches = ExpectedMatches;
+    FoundMatches.clear();
+
+    EXPECT_TRUE(FoundMatches.empty());
+  }
+
+private:
+  static std::string getMatchesAsString(const std::vector<Match> &Matches) {
+    if (Matches.empty())
+      return " none\n";
+    std::string FormattedMatches{"\n"};
+    for (const Match &M : Matches)
+      FormattedMatches += "\t" + M.getAsString() + ",\n";
+
+    return FormattedMatches;
+  }
+  static std::string getMatchComparisonText(const std::vector<Match> &Matches,
+                                            const T *const Node,
+                                            const ASTContext &Context) {
+    if (Matches.empty())
+      return " none\n";
+    std::string MatchStrings{"\n"};
+    for (const Match &M : Matches)
+      MatchStrings += llvm::formatv(
+          "\tMatchKind: {0}: '{1}' vs '{2}',\n", toString(M.getMatchKind()),
+          Match::getMatchText(Node, Context, M.getMatchKind(), false)
+              .value_or("<unknown>"),
+          M.getMatchString());
+
+    return MatchStrings;
+  }
+
+  static std::string getPossibleMatchStrings(const T *Node,
+                                             const ASTContext &Context) {
+    std::string MatchStrings{"\n"};
+    for (const auto Kind :
+         {MatchKind::Code, MatchKind::Name, MatchKind::TypeStr})
+      MatchStrings +=
+          llvm::formatv("\tMatchKind:  {0}: '{1}',\n", toString(Kind),
+                        Match::getMatchText(Node, Context, Kind, false)
+                            .value_or("<unknown>"))
+              .str();
+    return MatchStrings;
+  }
+
+  const std::string Id;
+  const std::vector<Match> ExpectedMatches;
+  std::vector<Match> Matches;
+  std::vector<Match> FoundMatches{};
+};
+
 class ASTMatchersTest : public ::testing::Test,
                         public ::testing::WithParamInterface<TestClangConfig> {
 protected:
@@ -458,6 +841,40 @@ class ASTMatchersTest : public ::testing::Test,
   }
 };
 
+class ASTMatchersDocTest
+    : public ::testing::Test,
+      public ::testing::WithParamInterface<TestClangConfig> {
+protected:
+  template <typename T>
+  testing::AssertionResult
+  matches(const Twine &Code, const T &AMatcher,
+          std::unique_ptr<BoundNodesCallback> FindResultVerifier,
+          const ArrayRef<std::string> CompileArgs = {},
+          const FileContentMappings &VirtualMappedFiles = {}) {
+    const TestClangConfig &TestConfig = GetParam();
+
+    auto Args = TestConfig.getCommandLineArgs();
+    Args.insert(Args.end(), CompileArgs.begin(), CompileArgs.end());
+    return clang::ast_matchers::matchAndVerifyResultConditionally(
+        Code, AMatcher, std::move(FindResultVerifier), /*ExpectMatch=*/true,
+        Args, getFilenameForTesting(TestConfig.Language), VirtualMappedFiles);
+  }
+
+  template <typename T>
+  testing::AssertionResult
+  notMatches(const Twine &Code, const T &AMatcher,
+             std::unique_ptr<BoundNodesCallback> FindResultVerifier,
+             const ArrayRef<std::string> CompileArgs = {},
+             const FileContentMappings &VirtualMappedFiles = {}) {
+    const TestClangConfig &TestConfig = GetParam();
+
+    auto Args = TestConfig.getCommandLineArgs();
+    Args.insert(Args.begin(), CompileArgs.begin(), CompileArgs.end());
+    return clang::ast_matchers::matchAndVerifyResultConditionally(
+        Code, AMatcher, std::move(FindResultVerifier), /*ExpectMatch=*/false,
+        Args, getFilenameForTesting(TestConfig.Language), VirtualMappedFiles);
+  }
+};
 } // namespace ast_matchers
 } // namespace clang
 
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index ec0be27774d8b..1d18869a6b8af 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -3082,10 +3082,13 @@ B func1() { return 42; }
     auto M = expr(unless(integerLiteral(equals(24)))).bind("intLit");
     EXPECT_TRUE(matchAndVerifyResultTrue(
         Code, traverse(TK_AsIs, M),
-        std::make_unique<VerifyIdIsBoundTo<Expr>>("intLit", 6)));
+        std::make_unique<VerifyIdIsBoundTo<Expr>>("intLit", 6),
+        {"-std=c++11"}));
+
     EXPECT_TRUE(matchAndVerifyResultTrue(
         Code, traverse(TK_IgnoreUnlessSpelledInSource, M),
-        std::make_unique<VerifyIdIsBoundTo<Expr>>("intLit", 1)));
+        std::make_unique<VerifyIdIsBoundTo<Expr>>("intLit", 1),
+        {"-std=c++11"}));
   }
   {
     auto M =
@@ -3128,7 +3131,8 @@ B func1() { return 42; }
     auto M = expr().bind("allExprs");
     EXPECT_TRUE(matchAndVerifyResultTrue(
         Code, traverse(TK_AsIs, M),
-        std::make_unique<VerifyIdIsBoundTo<Expr>>("allExprs", 6)));
+        std::make_unique<VerifyIdIsBoundTo<Expr>>("allExprs", 6),
+        {"-std=c++11"}));
     EXPECT_TRUE(matchAndVerifyResultTrue(
         Code, traverse(TK_IgnoreUnlessSpelledInSource, M),
         std::make_unique<VerifyIdIsBoundTo<Expr>>("allExprs", 1)));
@@ -5662,7 +5666,6 @@ TEST(HasParent, MatchesAllParents) {
 TEST(HasParent, NoDuplicateParents) {
   class HasDuplicateParents : public BoundNodesCallback {
   public:
-    bool run(const BoundNodes *Nodes) override { return false; }
     bool run(const BoundNodes *Nodes, ASTContext *Context) override {
       const Stmt *Node = Nodes->getNodeAs<Stmt>("node");
       std::set<const void *> Parents;
@@ -5871,16 +5874,14 @@ template <typename T> class VerifyMatchOnNode : public BoundNodesCallback {
 public:
   VerifyMatchOnNode(StringRef Id, const internal::Matcher<T> &InnerMatcher,
                     StringRef InnerId)
-    : Id(Id), InnerMatcher(InnerMatcher), InnerId(InnerId) {
-  }
-
-  bool run(const BoundNodes *Nodes) override { return false; }
+      : Id(Id), InnerMatcher(InnerMatcher), InnerId(InnerId) {}
 
   bool run(const BoundNodes *Nodes, ASTContext *Context) override {
     const T *Node = Nodes->getNodeAs<T>(Id);
     return selectFirst<T>(InnerId, match(InnerMatcher, *Node, *Context)) !=
-      nullptr;
+           nullptr;
   }
+
 private:
   std::string Id;
   internal::Matcher<T> InnerMatcher;
@@ -6074,7 +6075,7 @@ namespace {
 class ForCallablePreservesBindingWithMultipleParentsTestCallback
     : public BoundNodesCallback {
 public:
-  bool run(const BoundNodes *BoundNodes) override {
+  bool run(const BoundNodes *BoundNodes, ASTContext *Context) override {
     FunctionDecl const *FunDecl =
         BoundNodes->getNodeAs<FunctionDecl>("funDecl");
     // Validate test assumptions. This would be expressed as ASSERT_* in
@@ -6111,10 +6112,6 @@ class ForCallablePreservesBindingWithMultipleParentsTestCallback
     return true;
   }
 
-  bool run(const BoundNodes *BoundNodes, ASTContext *Context) override {
-    return run(BoundNodes);
-  }
-
 private:
   void ExpectCorrectResult(StringRef LogInfo,
                            ArrayRef<BoundNodes> Results) const {
diff --git a/clang/unittests/ASTMatchers/CMakeLists.txt b/clang/unittests/ASTMatchers/CMakeLists.txt
index 6a1e629d81b65..4867a2e39150b 100644
--- a/clang/unittests/ASTMatchers/CMakeLists.txt
+++ b/clang/unittests/ASTMatchers/CMakeLists.txt
@@ -4,11 +4,26 @@ set(LLVM_LINK_COMPONENTS
   TargetParser
   )
 
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ASTMatchersDocTests.cpp
+  COMMAND
+    ${CLANG_SOURCE_DIR}/utils/generate_ast_matcher_doc_tests.py ARGS
+    --input-file
+    ${CLANG_SOURCE_DIR}/include/clang/ASTMatchers/ASTMatchers.h
+    --output-file
+    ${CMAKE_CURRENT_BINARY_DIR}/ASTMatchersDocTests.cpp
+  MAIN_DEPENDENCY
+    ${CLANG_SOURCE_DIR}/include/clang/ASTMatchers/ASTMatchers.h
+  DEPENDS
+    ${CLANG_SOURCE_DIR}/utils/generate_ast_matcher_doc_tests.py ARGS
+)
+
 add_clang_unittest(ASTMatchersTests
   ASTMatchersInternalTest.cpp
   ASTMatchersNodeTest.cpp
   ASTMatchersNarrowingTest.cpp
   ASTMatchersTraversalTest.cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/ASTMatchersDocTests.cpp
   GtestMatchersTest.cpp
   )
 
diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp
index a2e960f143111..30b051e747f92 100644
--- a/clang/unittests/Interpreter/InterpreterTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterTest.cpp
@@ -381,4 +381,26 @@ TEST_F(InterpreterTest, Value) {
   EXPECT_TRUE(V9.isManuallyAlloc());
 }
 
+TEST_F(InterpreterTest, TranslationUnit_CanonicalDecl) {
+  std::vector<const char *> Args;
+  std::unique_ptr<Interpreter> Interp = createInterpreter(Args);
+
+  Sema &sema = Interp->getCompilerInstance()->getSema();
+
+  llvm::cantFail(Interp->ParseAndExecute("int x = 42;"));
+
+  TranslationUnitDecl *TU =
+      sema.getASTContext().getTranslationUnitDecl()->getCanonicalDecl();
+
+  llvm::cantFail(Interp->ParseAndExecute("long y = 84;"));
+
+  EXPECT_EQ(TU,
+            sema.getASTContext().getTranslationUnitDecl()->getCanonicalDecl());
+
+  llvm::cantFail(Interp->ParseAndExecute("char z = 'z';"));
+
+  EXPECT_EQ(TU,
+            sema.getASTContext().getTranslationUnitDecl()->getCanonicalDecl());
+}
+
 } // end anonymous namespace
diff --git a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
index 37e3546dc9087..4ff5e8b65a686 100644
--- a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
@@ -88,8 +88,12 @@ class BuildSyntaxTreeTest : public SyntaxTreeTest {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(SyntaxTreeTests, BuildSyntaxTreeTest,
-                        testing::ValuesIn(allTestClangConfigs()) );
+INSTANTIATE_TEST_SUITE_P(
+    SyntaxTreeTests, BuildSyntaxTreeTest,
+    testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 TEST_P(BuildSyntaxTreeTest, Simple) {
   EXPECT_TRUE(treeDumpEqual(
diff --git a/clang/unittests/Tooling/Syntax/MutationsTest.cpp b/clang/unittests/Tooling/Syntax/MutationsTest.cpp
index 1c3d6aac7183b..35692fd52181a 100644
--- a/clang/unittests/Tooling/Syntax/MutationsTest.cpp
+++ b/clang/unittests/Tooling/Syntax/MutationsTest.cpp
@@ -54,8 +54,11 @@ class MutationTest : public SyntaxTreeTest {
   };
 };
 
-INSTANTIATE_TEST_SUITE_P(SyntaxTreeTests, MutationTest,
-                        ::testing::ValuesIn(allTestClangConfigs()) );
+INSTANTIATE_TEST_SUITE_P(
+    SyntaxTreeTests, MutationTest, ::testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 TEST_P(MutationTest, RemoveStatement_InCompound) {
   CheckTransformation(RemoveStatement, "void test() { [[100+100;]] test(); }",
diff --git a/clang/unittests/Tooling/Syntax/SynthesisTest.cpp b/clang/unittests/Tooling/Syntax/SynthesisTest.cpp
index be8851267532c..ccfdcd05c7848 100644
--- a/clang/unittests/Tooling/Syntax/SynthesisTest.cpp
+++ b/clang/unittests/Tooling/Syntax/SynthesisTest.cpp
@@ -38,8 +38,11 @@ class SynthesisTest : public SyntaxTreeTest {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(SynthesisTests, SynthesisTest,
-                        ::testing::ValuesIn(allTestClangConfigs()) );
+INSTANTIATE_TEST_SUITE_P(
+    SynthesisTests, SynthesisTest, ::testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 TEST_P(SynthesisTest, Leaf_Punctuation) {
   buildTree("", GetParam());
diff --git a/clang/unittests/Tooling/Syntax/TreeTest.cpp b/clang/unittests/Tooling/Syntax/TreeTest.cpp
index 44cf42fa944a2..b553f704ae00c 100644
--- a/clang/unittests/Tooling/Syntax/TreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TreeTest.cpp
@@ -103,8 +103,11 @@ class TreeTest : public SyntaxTreeTest {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(TreeTests, TreeTest,
-                        ::testing::ValuesIn(allTestClangConfigs()) );
+INSTANTIATE_TEST_SUITE_P(
+    TreeTests, TreeTest, ::testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 TEST_P(TreeTest, FirstLeaf) {
   buildTree("", GetParam());
@@ -221,8 +224,11 @@ class ListTest : public SyntaxTreeTest {
   }
 };
 
-INSTANTIATE_TEST_SUITE_P(TreeTests, ListTest,
-                        ::testing::ValuesIn(allTestClangConfigs()) );
+INSTANTIATE_TEST_SUITE_P(
+    TreeTests, ListTest, ::testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
 
 /// "a, b, c"  <=> [("a", ","), ("b", ","), ("c", null)]
 TEST_P(ListTest, List_Separated_WellFormed) {
diff --git a/clang/unittests/Tooling/Syntax/TreeTestBase.cpp b/clang/unittests/Tooling/Syntax/TreeTestBase.cpp
index f387b503f3368..14c446c199906 100644
--- a/clang/unittests/Tooling/Syntax/TreeTestBase.cpp
+++ b/clang/unittests/Tooling/Syntax/TreeTestBase.cpp
@@ -48,8 +48,11 @@ ArrayRef<syntax::Token> tokens(syntax::Node *N,
 
 std::vector<TestClangConfig> clang::syntax::allTestClangConfigs() {
   std::vector<TestClangConfig> all_configs;
-  for (TestLanguage lang : {Lang_C89, Lang_C99, Lang_CXX03, Lang_CXX11,
-                            Lang_CXX14, Lang_CXX17, Lang_CXX20}) {
+  for (TestLanguage lang : {
+#define TESTLANGUAGE(lang, version, std_flag, version_index)                   \
+  Lang_##lang##version,
+#include "clang/Testing/TestLanguage.def"
+       }) {
     TestClangConfig config;
     config.Language = lang;
     config.Target = "x86_64-pc-linux-gnu";
diff --git a/clang/utils/analyzer/SATestBuild.py b/clang/utils/analyzer/SATestBuild.py
index bc86ed8b64e0e..66e1ab72985cd 100644
--- a/clang/utils/analyzer/SATestBuild.py
+++ b/clang/utils/analyzer/SATestBuild.py
@@ -176,7 +176,7 @@ def stdout(message: str):
 CHECKERS = ",".join(
     [
         "alpha.unix.SimpleStream",
-        "alpha.security.taint",
+        "optin.taint",
         "cplusplus.NewDeleteLeaks",
         "core",
         "cplusplus",
diff --git a/clang/utils/generate_ast_matcher_doc_tests.py b/clang/utils/generate_ast_matcher_doc_tests.py
new file mode 100755
index 0000000000000..2ee40d8a66068
--- /dev/null
+++ b/clang/utils/generate_ast_matcher_doc_tests.py
@@ -0,0 +1,1097 @@
+#!/usr/bin/env python3
+R"""
+TLDR:
+The order for a single code snippet example is:
+
+  \header{a.h}
+  \endheader     <- zero or more header
+
+  \code
+    int a = 42;
+  \endcode
+  \compile_args{-std=c++,c23-or-later} <- optional, supports std ranges and
+                                          whole languages
+
+  \matcher{expr()} <- one or more matchers in succession
+  \match{42}   <- one ore more matches in succession
+
+  \matcher{varDecl()} <- new matcher resets the context, the above
+                         \match will not count for this new
+                         matcher(-group)
+  \match{int a  = 42} <- only applies to the previous matcher (no the
+                         previous case)
+
+The above block can be repeated inside of a doxygen command for multiple
+code examples.
+
+Language Grammar:
+  [] denotes an optional, and <> denotes user-input
+
+  compile_args j:= \compile_args{[<compile_arg>;]<compile_arg>}
+  matcher_tag_key ::= type
+  match_tag_key ::= type || std || count
+  matcher_tags ::= [matcher_tag_key=<value>;]matcher_tag_key=<value>
+  match_tags ::= [match_tag_key=<value>;]match_tag_key=<value>
+  matcher ::= \matcher{[matcher_tags$]<matcher>}
+  matchers ::= [matcher] matcher
+  match ::= \match{[match_tags$]<match>}
+  matches ::= [match] match
+  case ::= matchers matches
+  cases ::= [case] case
+  header-block ::= \header{<name>} <code> \endheader
+  code-block ::= \code <code> \endcode
+  testcase ::= code-block [compile_args] cases
+
+The 'std' tag and '\compile_args' support specifying a specific
+language version, a whole language and all of it's versions, and thresholds
+(implies ranges). Multiple arguments are passed with a ',' seperator.
+For a language and version to execute a tested matcher, it has to match
+the specified '\compile_args' for the code, and the 'std' tag for the matcher.
+Predicates for the 'std' compiler flag are used with disjunction between
+languages (e.g. 'c || c++') and conjunction for all predicates specific
+to each language (e.g. 'c++11-or-later && c++23-or-earlier').
+
+Examples:
+ - c                                    all available versions of C
+ - c++11                                only C++11
+ - c++11-or-later                       C++11 or later
+ - c++11-or-earlier                     C++11 or earlier
+ - c++11-or-later,c++23-or-earlier,c    all of C and C++ between 11 and
+                                          23 (inclusive)
+ - c++11-23,c                             same as above
+
+Tags:
+
+  Type:
+  Match types are used to select where the string that is used to check if
+  a node matches comes from.
+  Available: code, name, typestr, typeofstr.
+  The default is 'code'.
+
+  Matcher types are used to mark matchers as submatchers with 'sub' or as
+  deactivated using 'none'. Testing submatchers is not implemented.
+
+  Count:
+  Specifying a 'count=n' on a match will result in a test that requires that
+  the specified match will be matched n times. Default is 1.
+
+  Std:
+  A match allows specifying if it matches only in specific language versions.
+  This may be needed when the AST differs between language versions.
+"""
+
+import re
+from argparse import ArgumentParser, Namespace
+from collections import defaultdict
+from enum import Enum
+from functools import reduce
+from pathlib import Path
+from sys import exit
+
+statistics = defaultdict(int)
+
+expected_failure_statistics = {
+    "missing_tests": 10,
+    "skipped_objc": 42,
+    "none_type_matchers": 6,
+}
+
+found_issue = False
+
+
+def parse_arguments() -> Namespace:
+    parser = ArgumentParser(
+        description="extract test cases from the documentation comments of AST"
+        " matchers",
+    )
+
+    parser.add_argument(
+        "--output-file",
+        help="The path to the ASTMatchersDocTest.cpp file where the tests are"
+        " written to.",
+        type=Path,
+    )
+
+    parser.add_argument(
+        "--input-file",
+        help="The path to the ASTMatchers.h file",
+        type=Path,
+    )
+
+    return parser.parse_args()
+
+
+def matcher_to_node_type(matcher: str) -> str:
+    """Converts the written top-level matcher to the node type that the matcher
+    will bind.
+    Contains special cases because not all matcher names can be simply converted
+    to a type by spelling it with uppercase characters in the beginning.
+
+    Args:
+        matcher: The matcher string
+
+    Returns:
+        The type of the node that a match will bind to
+    """
+    if matcher.startswith("traverse"):
+        comma_loc = matcher.find(",")
+        return matcher_to_node_type(matcher[comma_loc + 1 :].strip())
+    if matcher.startswith("cxxBoolLiteral"):
+        return "CXXBoolLiteralExpr"
+    if matcher.startswith("binaryOperation"):
+        return "Expr"
+    if matcher.startswith("invocation"):
+        return "Expr"
+    if matcher.startswith("alignOfExpr"):
+        return "Expr"
+    if matcher.startswith("sizeOfExpr"):
+        return "Expr"
+    if matcher.startswith("floatLiteral"):
+        return "FloatingLiteral"
+    if matcher.startswith("gnuNullExpr"):
+        return "GNUNullExpr"
+    if matcher.startswith("cxx"):
+        return "CXX" + matcher[3 : matcher.find("(")]
+    if matcher.startswith("omp"):
+        return "OMP" + matcher[3 : matcher.find("(")]
+    if matcher.startswith("cuda"):
+        return "CUDA" + matcher[4 : matcher.find("(")]
+    if matcher.startswith("objc"):
+        return "ObjC" + matcher[4 : matcher.find("(")]
+    return matcher[0:1].upper() + matcher[1 : matcher.find("(")]
+
+
+def get_clang_config_constraint_expr(std: str) -> str:
+    """Converts a single argument to 'std' into the corresponding check to be
+    done in the tests.
+
+    Args:
+        std: A single argument to 'std' (e.g. 'c++11-or-later')
+
+    Returns:
+        An expression that checks is the test config enables what the 'std'
+        argument specifies.
+    """
+    if std == "":
+        return ""
+
+    or_later_str = "-or-later"
+    or_earlier_str = "-or-earlier"
+    match = re.match(r"c(\d\d)-?(\d\d)?", std)
+    if match:
+        if len(match.groups()) == 3:
+            return f"Conf.isCOrLater({match.group(1)}) && Conf.isOrEarlier({match.group(2)})"
+        if std.endswith(or_later_str):
+            return f"Conf.isCOrLater({match.group(1)})"
+        if std.endswith(or_earlier_str):
+            return f"Conf.isCOrEarlier({match.group(1)})"
+        return f"Conf.Language == Lang_C{match.group(1)}"
+
+    match = re.match(r"c\+\+(\d\d)-?(\d\d)?", std)
+    if match:
+        if len(match.groups()) == 3:
+            return f"Conf.isCXXOrLater({match.group(1)}) && Conf.isCXXOrEarlier({match.group(2)})"
+        if std.endswith(or_later_str):
+            return f"Conf.isCXXOrLater({match.group(1)})"
+        if std.endswith(or_earlier_str):
+            return f"Conf.isCXXOrEarlier({match.group(1)})"
+        return f"Conf.Language == Lang_CXX{match.group(1)}"
+
+    if std == "c":
+        return "Conf.isC()"
+
+    if std == "c++":
+        return "Conf.isCXX()"
+
+    if std.startswith("-ObjC"):
+        return ""
+
+    return ""
+
+
+class TestLanguage:
+    """Wraps multiple args to 'std' for emitting the config check inside the
+    tests.
+
+    Attributes:
+        raw: The arg to 'std' as written
+        c: The config check expression for C
+        cxx: The config check expression for C++
+        objc: The config check expression for ObjC
+    """
+
+    def __init__(self, std: str) -> None:
+        self.raw = std
+        self.c = ""
+        self.cxx = ""
+        self.objc = ""
+        if std.startswith("-ObjC"):
+            self.objc = std
+            return
+
+        for standard_spec in std.split(","):
+            expr = get_clang_config_constraint_expr(standard_spec)
+            if standard_spec.startswith("c++"):
+                if self.cxx != "":
+                    self.cxx += " && "
+                self.cxx += expr
+            elif standard_spec.startswith("c"):
+                if self.c != "":
+                    self.c += " && "
+                self.c += expr
+
+    def has_value(self):
+        return self.c != "" or self.cxx != "" or self.objc != ""
+
+    def get_config_check_expr(self):
+        if self.c != "" and self.cxx != "":
+            return f"({self.c}) || ({self.cxx})"
+        if self.c != "":
+            return self.c
+        if self.cxx != "":
+            return self.cxx
+        if self.objc != "":
+            return self.objc
+        return ""
+
+
+class Tags:
+    """Wrapper to parse and store the tags that can be passed to '\\match' and
+    '\\matcher'.
+
+    Attributes:
+        map: The parsed tags by key ('str' -> 'str' map)
+        opt_test_language: The 'std' tag is handled by TestLanguage
+    """
+
+    def __init__(self, tag_string: str) -> None:
+        self.map = defaultdict(
+            lambda: "",
+            [split_first(tag, "=") for tag in tag_string.split(";")],
+        )
+
+        self.opt_test_language = None
+        if "std" in self.map:
+            self.opt_test_language = TestLanguage(self.map["std"])
+            self.map.pop("std")
+
+
+class Matcher:
+    def __init__(self, matcher: str, tags: Tags) -> None:
+        self.matcher = matcher
+        self.tags = tags
+
+    def is_sub_matcher(self) -> bool:
+        return self.tags.map["type"] == "sub"
+
+    def __format__(self, format_spec: str) -> str:
+        return self.matcher.__format__(format_spec)
+
+    def __str__(self) -> str:
+        return f"{self.matcher}"
+
+    def __repr__(self) -> str:
+        return f"{self.matcher}"
+
+
+class Match:
+    def __init__(self, match_str: str, tags: Tags) -> None:
+        self.match_str = match_str.replace("\n", " ")
+        self.tags = tags
+
+    def is_sub_match(self) -> bool:
+        return self.tags.map["sub"] != ""
+
+    def __format__(self, format_spec: str) -> str:
+        return self.match_str.__format__(format_spec)
+
+    def __str__(self) -> str:
+        return f"{self.match_str}"
+
+    def __repr__(self) -> str:
+        return f"{self.match_str}"
+
+    def get_as_cpp_raw_string(self) -> str:
+        return f'R"cpp({self.match_str})cpp"'
+
+
+def get_lang_spec_and_remove_from_list(args: list) -> str:
+    for arg in args:
+        if arg == "-ObjC":
+            args.remove(arg)
+            return arg
+        if arg.startswith("-std="):
+            args.remove(arg)
+            return arg[5:]
+    return ""
+
+
+class CompileArgs:
+    """Represents the '\\compile_args' command and its arguments.
+
+    Attributes:
+        lang_spec: The specified test language
+        args: All other arguments
+    """
+
+    def __init__(self, args: list) -> None:
+        self.lang_spec = TestLanguage(get_lang_spec_and_remove_from_list(args))
+        self.args = args
+
+        if any(("cuda" in arg for arg in self.args)) and not any(
+            "-x" in arg for arg in self.args
+        ):
+            self.args.append("-xcuda")
+            self.args.append("-nocudainc")
+            self.args.append("-nocudalib")
+
+    def is_cuda(self) -> bool:
+        return any("cuda" in cmd for cmd in self.args)
+
+    def is_objc(self) -> bool:
+        return self.lang_spec.objc != ""
+
+
+def get_any_valid_std_specified(lang_spec: str) -> str:
+    """Get any argument to '-std' that satisfies the language specification
+    in 'lang_spec'
+
+    Args:
+        lang_spec: An argument to 'std'
+
+    Returns:
+        Any valid argument to the '-std' compiler flag that satisfies
+        'lang_spec'
+    """
+    if lang_spec == "":
+        return "c++11"
+
+    first_comma = lang_spec.find(",")
+    if first_comma != -1:
+        lang_spec = lang_spec[:first_comma]
+
+    first_minus = lang_spec.find("-")
+    if first_minus != -1:
+        lang_spec = lang_spec[:first_minus]
+
+    elif lang_spec == "c":
+        lang_spec = "c11"
+    elif lang_spec == "c++":
+        lang_spec = "c++11"
+
+    return lang_spec
+
+
+def get_with_lang_spec(args: CompileArgs) -> list:
+    """Construct compiler arguments from a CompileArgs instance
+
+    Args:
+        args: The arguments to '\\compile_args'
+
+    Returns:
+        A list of compiler arguments that satisfy what is specified in 'args'
+    """
+    if args.lang_spec.objc.startswith("-ObjC"):
+        return [*args.args, args.lang_spec.objc]
+
+    if args.lang_spec.has_value():
+        return [*args.args, "-std=" + get_any_valid_std_specified(args.lang_spec.raw)]
+
+    return args.args
+
+
+cuda_header: str = """
+    typedef unsigned long long size_t;
+    #define __constant__ __attribute__((constant))
+    #define __device__ __attribute__((device))
+    #define __global__ __attribute__((global))
+    #define __host__ __attribute__((host))
+    #define __shared__ __attribute__((shared))
+    struct dim3 {
+    unsigned x, y, z;
+    __host__ __device__ dim3(unsigned x, unsigned y = 1, unsigned z = 1)
+        : x(x), y(y), z(z) {}
+    };
+    typedef struct cudaStream *cudaStream_t;
+    int cudaConfigureCall(dim3 gridSize, dim3 blockSize,
+                        size_t sharedSize = 0,
+                        cudaStream_t stream = 0);
+    extern "C" unsigned __cudaPushCallConfiguration(
+      dim3 gridDim, dim3 blockDim, size_t sharedMem = 0, void *stream =
+    0);
+"""
+
+
+class MatchType(Enum):
+    """Available types of a match, specified by using the lowercase version in
+    doxygen: '\\match{type=typestr}'.
+    Specifies what string from the matched node should be used to check for a
+    match.
+
+    Attributes:
+        Invalid: Specified an invalid match type
+        Code: Check if a matched node matches by using it's text
+              (SourceRange -> Lexer)
+        Name: Use the name of the matched node to check if it matches
+        TypeStr: Use the string representation of the matched type to check if
+                 it matches
+    """
+
+    Invalid = 0
+    Code = 1
+    Name = 2
+    TypeStr = 3
+
+
+def get_match_type(match: Match) -> MatchType:
+    match_type = match.tags.map["type"]
+    if match_type == "name":
+        return MatchType.Name
+    if match_type == "" or match_type == "code":
+        return MatchType.Code
+    if match_type == "typestr":
+        return MatchType.TypeStr
+    print(f"match {match} has an invalid match type: {match_type}")
+    statistics["match_type_invalid"] += 1
+    return MatchType.Invalid
+
+
+def get_match_count(match: Match) -> str:
+    count = match.tags.map["count"]
+    if count == "":
+        return "1"
+    return count
+
+
+def construct_match(match: Match) -> str:
+    result = ""
+    if match.tags.opt_test_language:
+        result += f"\tif ({match.tags.opt_test_language.get_config_check_expr()})\n\t"
+
+    result += f"\tMatches.emplace_back(MatchKind::{get_match_type(match).name}, {match.get_as_cpp_raw_string()}, {get_match_count(match)});"
+    return result
+
+
+class TestCase:
+    """Represents a single code example and its tests. Tests are added during
+    parsing and finally emitted as C++ test code.
+
+    Attributes:
+        input_file: The file path where the matcher was specified
+        line: The line that the test case starts at
+        cases: The collected cases for this example code
+        code: The example code
+        headers: Headers, if any
+        compile_args: Compile arguments, if any
+    """
+
+    def __init__(self, input_file: Path, line: int):
+        self.input_file = input_file
+        self.line = line
+        self.cases: list[tuple[list[Matcher], list[Match]]] = []
+        self.code: str = ""
+        self.headers: list[tuple[str, str]] = []
+        self.compile_args = None
+
+    def get_last_case(self) -> tuple:
+        return self.cases[-1]
+
+    def add_case(self):
+        self.cases.append(([], []))
+
+    def add_matcher(self, matcher: Matcher):
+        self.get_last_case()[0].append(matcher)
+
+    def add_match(self, match: Match):
+        self.get_last_case()[1].append(match)
+
+    def has_non_lang_spec_compile_args(self) -> bool:
+        return self.compile_args is not None and len(self.compile_args.args) != 0
+
+    def has_headers(self) -> bool:
+        return len(self.headers) != 0
+
+    def get_compile_commands(self) -> str:
+        if self.compile_args is None:
+            return "{}"
+        return "{ " + ",".join([f'"{arg}"' for arg in self.compile_args.args]) + " }"
+
+    def diag(self, message: str):
+        print(f"{self.input_file}:{self.line + 1}: {message}")
+
+    def has_match_with_std_constraint(self) -> bool:
+        return any(
+            (
+                any(match.tags.opt_test_language for match in case[1])
+                for case in self.cases
+            ),
+        )
+
+    def get_formated_headers(self) -> str:
+        return "\n\t".join(
+            f'std::pair{{"{header[0]}", \n\tR"cpp({header[1]})cpp"}},'
+            for header in self.headers
+        )
+
+    def build_test_case(self):
+        self.code = self.code.strip("\n")
+        has_cuda = self.compile_args and self.compile_args.is_cuda()
+        if has_cuda:
+            self.headers.append(("cuda.h", cuda_header))
+
+        res = ""
+        if has_cuda:
+            res += "#if LLVM_HAS_NVPTX_TARGET\n"
+
+        res += f"""TEST_P(ASTMatchersDocTest, docs_{self.line + 1}) {{
+    const StringRef Code = R"cpp(\n"""
+        if has_cuda:
+            res += '\t#include "cuda.h"\n'
+        res += f'{self.code})cpp";\n'
+
+        if self.has_headers():
+            res += f"\tconst FileContentMappings VirtualMappedFiles = {{{self.get_formated_headers()}}};"
+
+        if self.compile_args and self.compile_args.lang_spec.objc != "":
+            statistics["skipped_objc"] += 1
+            return ""
+
+        statistics["code_snippets"] += 1
+
+        statistics["matches"] += reduce(
+            lambda a, b: a + b,
+            (len(v[1]) for v in self.cases),
+        )
+
+        statistics["matchers"] += reduce(
+            lambda a, b: a + b,
+            (len(v[0]) for v in self.cases),
+        )
+
+        if self.compile_args and self.compile_args.lang_spec.has_value():
+            res += f"""
+    const TestClangConfig& Conf = GetParam();
+    const bool ConfigEnablesCheckingCode = {self.compile_args.lang_spec.get_config_check_expr()};
+    if (!ConfigEnablesCheckingCode) return;\n"""
+
+        elif self.has_match_with_std_constraint():
+            res += "\n\tconst TestClangConfig& Conf = GetParam();\n"
+
+        # Don't want to emit a test without any actual tested matchers.
+        # Would result in warnings about unused variables (Code) during the
+        # build.
+        case_has_test = False
+
+        for matchers, matches in self.cases:
+            if len(matchers) == 0:
+                self.diag("test cases have no matchers")
+                continue
+
+            for matcher in matchers:
+                if matcher.matcher.startswith("objc"):
+                    statistics["skipped_objc"] += 1
+                    continue
+                # FIXME: add support for testing submatchers
+                if matcher.tags.map["type"] == "sub":
+                    continue
+                if matcher.tags.map["type"] == "none":
+                    statistics["none_type_matchers"] += 1
+                    continue
+                matcher_type = matcher_to_node_type(matcher.matcher)
+
+                statistics["tested_matchers"] += 1
+                case_has_test = True
+
+                verifier_type = f"VerifyBoundNodeMatch<{matcher_type}>"
+
+                code_adding_matches = "\n".join(
+                    construct_match(match)
+                    for match in matches
+                    if match.tags.map["sub"] == ""
+                )
+
+                match_function = (
+                    "matches" if len(code_adding_matches) != 0 else "notMatches"
+                )
+
+                res += f"""
+    {{
+    SCOPED_TRACE("Test failure from docs_{self.line + 1} originates here.");
+    using Verifier = {verifier_type};
+
+    std::vector<Verifier::Match> Matches;
+{code_adding_matches}
+
+    EXPECT_TRUE({match_function}(
+        Code,
+        {matcher.matcher}.bind("match"),
+        std::make_unique<Verifier>("match", Matches)"""
+
+                if self.has_headers():
+                    res += f",\n\t\t{self.get_compile_commands()}"
+                    res += ",\n\t\tVirtualMappedFiles"
+                elif self.has_non_lang_spec_compile_args():
+                    res += f",\n\t\t{self.get_compile_commands()}"
+
+                res += "));\n\t}\n"
+
+        if not case_has_test:
+            return ""
+
+        res += "}"
+        if has_cuda:
+            res += "\n#endif\n"
+        return res
+
+
+class ParsingContext(Enum):
+    NoneCtx = 0
+    Code = 1
+    Header = 2
+    CompileArgs = 3
+    Matcher = 4
+    Match = 5
+    NoMatch = 6
+
+
+def split_first(data: str, delim: str) -> tuple:
+    pos = data.find(delim)
+    if pos == -1:
+        return (data, "")
+
+    return (data[0:pos], data[pos + 1 :])
+
+
+def find_matching_closing_rbrace(
+    data: str,
+    start_pos: int,
+    braces_to_be_matched: int,
+) -> int:
+    next_lbrace = data.find("{", start_pos)
+    next_rbrace = data.find("}", start_pos)
+    if next_lbrace != -1:
+        if next_lbrace < next_rbrace:
+            return find_matching_closing_rbrace(
+                data,
+                next_lbrace + 1,
+                braces_to_be_matched + 1,
+            )
+        if braces_to_be_matched == 0:
+            return next_rbrace
+        return find_matching_closing_rbrace(
+            data,
+            next_rbrace + 1,
+            braces_to_be_matched - 1,
+        )
+
+    if braces_to_be_matched > 0:
+        return find_matching_closing_rbrace(
+            data,
+            next_rbrace + 1,
+            braces_to_be_matched - 1,
+        )
+
+    return next_rbrace
+
+
+class CommentedMatcher:
+    """Represents a matcher and it's corresponding doxygen comment.
+
+    Attributes:
+        comment: The doxygen comment
+        matcher: The actual matcher code/implementation
+    """
+
+    def __init__(
+        self,
+        comment: list,
+        matcher: list,
+    ):
+        self.comment = comment
+        self.matcher = matcher
+
+
+class BlockParser:
+    """The parser that parses the test cases from a doxygen block.
+
+    Attributes:
+        input_file: The file where the matchers are located
+        cases: All TestCases
+        current_line: The line that the parser is currently on
+        data: The lines in the comment
+    """
+
+    def __init__(self, data: CommentedMatcher, input_file: Path):
+        self.input_file = input_file
+        self.cases = list()
+        self.current_line = data.comment[0][0]
+        self.data = "\n".join([line[1] for line in data.comment])
+        self.diagnostics = list()
+
+    def advance(self) -> ParsingContext:
+        """Find the next doxygen command to be parsed and return the
+        ParsingContext it represents.
+
+        Returns:
+            The ParsingContext of the location the parser arived at.
+        """
+        begin_tags = {
+            "\\code": ParsingContext.Code,
+            "\\header{": ParsingContext.Header,
+            "\\compile_args{": ParsingContext.CompileArgs,
+            "\\matcher{": ParsingContext.Matcher,
+            "\\match{": ParsingContext.Match,
+            "\\nomatch{": ParsingContext.NoMatch,
+        }
+
+        matches = list()
+
+        for tag, ctx in begin_tags.items():
+            match = self.data.find(tag)
+            if match == -1:
+                continue
+            matches.append((match, ctx, tag))
+
+        if len(matches) == 0:
+            return ParsingContext.NoneCtx
+
+        matches.sort()
+
+        loc, ctx, tag = matches[0]
+        loc = loc + len(tag) - 1
+        self.consume_until(loc)
+        return ctx
+
+    def add_case(self, line: int):
+        self.cases.append(TestCase(self.input_file, line))
+
+    def delete_last_test(self):
+        self.cases.pop()
+
+    def get_last_case(self) -> TestCase:
+        return self.cases[-1]
+
+    def add_matcher(self, matcher: Matcher):
+        if matcher.is_sub_matcher():
+            return
+        self.get_last_case().add_matcher(matcher)
+
+    def add_match(self, match: Match):
+        if match.is_sub_match():
+            return
+        self.get_last_case().add_match(match)
+
+    def consume_until(self, pos: int):
+        self.current_line += self.data.count("\n", 0, pos)
+        self.data = self.data[pos + 1 :]
+
+    def visit_code_block(self):
+        code_end = "\\endcode"
+        endcode_loc = self.data.find(code_end)
+        end = endcode_loc + len(code_end)
+
+        self.get_last_case().code = self.data[:endcode_loc].replace("\n", "\n\t")
+
+        self.consume_until(end)
+
+    def visit_header(self):
+        header_end = "\\endheader"
+        endheader_loc = self.data.find(header_end)
+        end = endheader_loc + len(header_end)
+
+        vrbrace_loc = find_matching_closing_rbrace(self.data, 0, 0)
+        header_name = self.data[0:vrbrace_loc]
+
+        self.get_last_case().headers.append(
+            (
+                header_name,
+                self.data[vrbrace_loc + 1 : endheader_loc].replace("\n", "\n\t"),
+            ),
+        )
+
+        self.consume_until(end)
+
+    def visit_compile_args(self):
+        end = find_matching_closing_rbrace(self.data, 0, 0)
+
+        args = self.data[:end]
+        self.consume_until(end)
+
+        self.get_last_case().compile_args = CompileArgs(
+            [split_arg for arg in args.split(";") for split_arg in arg.split(" ")],
+        )
+
+    def build_matcher(self):
+        end = find_matching_closing_rbrace(self.data, 0, 0)
+        tag_separator = self.data.find("$")
+        tags = Tags("")
+        if tag_separator != -1 and tag_separator < end:
+            tags = Tags(self.data[:tag_separator])
+            self.consume_until(tag_separator)
+
+            # find the '}' again, self.data shifted
+            end = find_matching_closing_rbrace(self.data, 0, 0)
+
+        matcher_str = self.data[:end]
+        if matcher_str.count("(") != matcher_str.count(")"):
+            self.diag(
+                f"The matcher {matcher_str} has an unbalanced number of parentheses"
+            )
+            self.consume_until(end)
+            return None
+
+        matcher = Matcher(self.data[:end], tags)
+        self.consume_until(end)
+        return matcher
+
+    def visit_match(self):
+        end = find_matching_closing_rbrace(self.data, 0, 0)
+        tag_separator = self.data.find("$")
+        tags = Tags("")
+        if tag_separator != -1 and tag_separator < end:
+            tags = Tags(self.data[:tag_separator])
+            self.consume_until(tag_separator)
+
+            # find the '}' again, self.data shifted
+            end = find_matching_closing_rbrace(self.data, 0, 0)
+        self.add_match(Match(self.data[:end], tags))
+        self.consume_until(end)
+
+    def diag(self, msg: str):
+        # Save diagnostics instead of emitting them to remove the noise of diagnosed issues,
+        # when the issues are all expected.
+        self.diagnostics.append(f"{self.input_file}:{self.current_line + 1}: {msg}")
+
+    def run(self):
+        ctx = self.advance()
+
+        if ctx == ParsingContext.NoneCtx:
+            self.diag("matcher is missing an example")
+            statistics["missing_tests"] += 1
+            return
+
+        while ctx != ParsingContext.NoneCtx:
+            self.add_case(self.current_line)
+            while ctx == ParsingContext.Header:
+                self.visit_header()
+                ctx = self.advance()
+
+            if ctx != ParsingContext.Code:
+                self.diag(f"expected {ParsingContext.Code}, not {ctx}")
+                statistics["missing_tests"] += 1
+                self.delete_last_test()
+                return
+
+            self.visit_code_block()
+            ctx = self.advance()
+
+            if ctx == ParsingContext.CompileArgs:
+                self.visit_compile_args()
+                compile_args = self.get_last_case().compile_args
+                ctx = self.advance()
+                if compile_args and compile_args.is_objc():
+                    self.delete_last_test()
+                    statistics["skipped_objc"] += 1
+
+                    while ctx != ParsingContext.Code and ctx != ParsingContext.NoneCtx:
+                        ctx = self.advance()
+
+                    continue
+
+            if ctx != ParsingContext.Matcher:
+                if ctx == ParsingContext.NoneCtx:
+                    self.diag(
+                        "this code example is missing an example matcher and matches",
+                    )
+                else:
+                    self.diag(
+                        f"expected {ParsingContext.Matcher} after {ParsingContext.Code}, not {ctx}",
+                    )
+
+                statistics["missing_tests"] += 1
+                self.delete_last_test()
+                return
+
+            while ctx == ParsingContext.Matcher:
+                matcher = self.build_matcher()
+                if matcher and not matcher.is_sub_matcher():
+                    self.get_last_case().add_case()
+
+                if matcher:
+                    self.add_matcher(matcher)
+
+                ctx = self.advance()
+                while ctx == ParsingContext.Matcher:
+                    matcher = self.build_matcher()
+                    if matcher:
+                        self.add_matcher(matcher)
+                    ctx = self.advance()
+
+                if ctx != ParsingContext.Match and ctx != ParsingContext.NoMatch:
+                    if ctx == ParsingContext.NoneCtx:
+                        self.diag(
+                            "this matcher does not specify what it should or shouldn't match",
+                        )
+                    else:
+                        self.diag(
+                            f"expected {ParsingContext.Match} or {ParsingContext.NoMatch} after {ParsingContext.Matcher}, not {ctx}",
+                        )
+
+                    statistics["matcher_groups_without_matches"] += 1
+                    break
+
+                while ctx == ParsingContext.Match or ctx == ParsingContext.NoMatch:
+                    if ctx == ParsingContext.Match:
+                        self.visit_match()
+                    ctx = self.advance()
+
+
+def parse_block(data: CommentedMatcher, input_file: Path) -> BlockParser:
+    parser = BlockParser(data, input_file)
+    parser.run()
+    return parser
+
+
+def parse(data: list, input_file: Path) -> tuple:
+    result: tuple = ([], [])
+
+    for parsed_block in [parse_block(block, input_file) for block in data]:
+        result[0].extend(parsed_block.cases)
+        result[1].extend(parsed_block.diagnostics)
+
+    return result
+
+
+def group_doc_comment_and_followed_code(
+    enumerated_lines: list,
+) -> list:
+    result: list = []
+
+    start_new_group_on_comment = True
+    for line_nr, line in enumerate(enumerated_lines, start=1):
+        if line.startswith("///"):
+            if start_new_group_on_comment:
+                result.append(CommentedMatcher([], []))
+            start_new_group_on_comment = False
+            if len(result) != 0:
+                result[-1].comment.append((line_nr, line[4:].rstrip()))
+        else:
+            start_new_group_on_comment = True
+            if len(result) != 0:
+                result[-1].matcher.append((line_nr, line))
+
+    return result
+
+
+test_file_begin = """
+// unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp - AST matcher unit tests//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTMatchersTest.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Testing/TestClangConfig.h"
+#include "gtest/gtest.h"
+#include <string>
+
+namespace clang {
+namespace ast_matchers {
+"""
+
+test_file_end = """
+static std::vector<TestClangConfig> allTestClangConfigs() {
+  std::vector<TestClangConfig> all_configs;
+  for (TestLanguage lang : {
+#define TESTLANGUAGE(lang, version, std_flag, index) Lang_##lang##version,
+#include "clang/Testing/TestLanguage.def"
+       }) {
+    TestClangConfig config;
+    config.Language = lang;
+
+    // Use an unknown-unknown triple so we don't instantiate the full system
+    // toolchain.  On Linux, instantiating the toolchain involves stat'ing
+    // large portions of /usr/lib, and this slows down not only this test, but
+    // all other tests, via contention in the kernel.
+    //
+    // FIXME: This is a hack to work around the fact that there's no way to do
+    // the equivalent of runToolOnCodeWithArgs without instantiating a full
+    // Driver.  We should consider having a function, at least for tests, that
+    // invokes cc1.
+    config.Target = "i386-unknown-unknown";
+    all_configs.push_back(config);
+
+    // Windows target is interesting to test because it enables
+    // `-fdelayed-template-parsing`.
+    config.Target = "x86_64-pc-win32-msvc";
+    all_configs.push_back(config);
+  }
+  return all_configs;
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    ASTMatchersTests, ASTMatchersDocTest,
+    testing::ValuesIn(allTestClangConfigs()),
+    [](const testing::TestParamInfo<TestClangConfig> &Info) {
+      return Info.param.toShortString();
+    });
+} // namespace ast_matchers
+} // namespace clang
+"""
+
+
+def main():
+    args = parse_arguments()
+
+    assert args.input_file.exists()
+
+    comment_blocks = group_doc_comment_and_followed_code(
+        args.input_file.read_text().split("\n"),
+    )
+
+    statistics["doxygen_blocks"] = len(comment_blocks)
+    results = parse(comment_blocks, args.input_file)
+
+    res: str = (
+        "\n\n".join(
+            filter(
+                lambda case: len(case) != 0,
+                [r.build_test_case() for r in results[0]],
+            ),
+        )
+        + "\n"
+    )
+
+    args.output_file.write_text(test_file_begin + res + test_file_end)
+
+    global found_issue
+    for key, value in expected_failure_statistics.items():
+        if value != statistics[key]:
+            print(
+                "Mismatch between expected and actual failure statistic: "
+                f"{key}: expected: {value}, actual: {statistics[key]}. "
+                "Please fix the issue or adjust the expected_failure_statistics"
+                " value if appropriate."
+            )
+            found_issue = True
+
+    if found_issue:
+        for diag in results[1]:
+            print(diag)
+
+        print("Statistics:")
+        for key, value in statistics.items():
+            print(f"\t{key: <30}: {value: >5}")
+
+        exit(1)
+
+
+# FIXME: add support in the html gen script to differentiate between overloads examples
+# FIXME: should verify that all polymorphic ast nodes have a test
+
+
+if __name__ == "__main__":
+    main()
diff --git a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
index ef90f02f57f9f..5c6c3475ed910 100644
--- a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
+++ b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
@@ -33,7 +33,7 @@ enum CPUFeatures {
   FEAT_FP,
   FEAT_SIMD,
   FEAT_CRC,
-  FEAT_SHA1,
+  RESERVED_FEAT_SHA1, // previously used and now ABI legacy
   FEAT_SHA2,
   FEAT_SHA3,
   FEAT_AES,
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
index f0694900f231f..da1c4dfcb0c9a 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
@@ -77,7 +77,6 @@ void __init_cpu_features_resolver(void) {
     CHECK_BIT(CAP_BIT_FEAT_RDM, FEAT_RDM);
     CHECK_BIT(CAP_BIT_FEAT_LSE, FEAT_LSE);
     CHECK_BIT(CAP_BIT_FEAT_SHA256, FEAT_SHA2);
-    CHECK_BIT(CAP_BIT_FEAT_SHA1, FEAT_SHA1);
     CHECK_BIT(CAP_BIT_FEAT_AES, FEAT_AES);
     CHECK_BIT(CAP_BIT_FEAT_PMULL, FEAT_PMULL);
     CHECK_BIT(CAP_BIT_FEAT_SPECRES, FEAT_PREDRES);
@@ -123,7 +122,6 @@ void __init_cpu_features_resolver(void) {
       {"hw.optional.arm.FEAT_LSE", FEAT_LSE},
       {"hw.optional.AdvSIMD", FEAT_SIMD},
       {"hw.optional.armv8_crc32", FEAT_CRC},
-      {"hw.optional.arm.FEAT_SHA1", FEAT_SHA1},
       {"hw.optional.arm.FEAT_SHA256", FEAT_SHA2},
       {"hw.optional.arm.FEAT_SHA3", FEAT_SHA3},
       {"hw.optional.arm.FEAT_AES", FEAT_AES},
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
index 1ae4780e4978e..b0c4c801a633a 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
@@ -24,8 +24,6 @@ void __init_cpu_features_resolver() {
     setCPUFeature(FEAT_AES);
   if (features & ZX_ARM64_FEATURE_ISA_PMULL)
     setCPUFeature(FEAT_PMULL);
-  if (features & ZX_ARM64_FEATURE_ISA_SHA1)
-    setCPUFeature(FEAT_SHA1);
   if (features & ZX_ARM64_FEATURE_ISA_SHA256)
     setCPUFeature(FEAT_SHA2);
   if (features & ZX_ARM64_FEATURE_ISA_CRC32)
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
index ccee4a274ea70..6f6bd19a2120a 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc
@@ -35,8 +35,6 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
     setCPUFeature(FEAT_RDM);
   if (hwcap & HWCAP_AES)
     setCPUFeature(FEAT_AES);
-  if (hwcap & HWCAP_SHA1)
-    setCPUFeature(FEAT_SHA1);
   if (hwcap & HWCAP_SHA2)
     setCPUFeature(FEAT_SHA2);
   if (hwcap & HWCAP_JSCVT)
diff --git a/compiler-rt/lib/rtsan/rtsan.cpp b/compiler-rt/lib/rtsan/rtsan.cpp
index 87c3611935ee5..f9741b4fe3509 100644
--- a/compiler-rt/lib/rtsan/rtsan.cpp
+++ b/compiler-rt/lib/rtsan/rtsan.cpp
@@ -18,6 +18,7 @@
 #include "sanitizer_common/sanitizer_atomic.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_mutex.h"
+#include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
 
 using namespace __rtsan;
@@ -49,7 +50,30 @@ static auto OnViolationAction(DiagnosticsInfo info) {
   return [info]() {
     IncrementTotalErrorCount();
 
-    PrintDiagnostics(info);
+    BufferedStackTrace stack;
+
+    // We use the unwind_on_fatal flag here because of precedent with other
+    // sanitizers, this action is not necessarily fatal if halt_on_error=false
+    stack.Unwind(info.pc, info.bp, nullptr,
+                 common_flags()->fast_unwind_on_fatal);
+
+    // If in the future we interop with other sanitizers, we will
+    // need to make our own stackdepot
+    StackDepotHandle handle = StackDepotPut_WithHandle(stack);
+
+    const bool is_stack_novel = handle.use_count() == 0;
+
+    // Marked UNLIKELY as if user is runing with halt_on_error=false
+    // we expect a high number of duplicate stacks. We are willing
+    // To pay for the first insertion.
+    if (UNLIKELY(is_stack_novel)) {
+      IncrementUniqueErrorCount();
+
+      PrintDiagnostics(info);
+      stack.Print();
+
+      handle.inc_use_count_unsafe();
+    }
 
     if (flags().halt_on_error)
       Die();
@@ -90,19 +114,19 @@ SANITIZER_INTERFACE_ATTRIBUTE bool __rtsan_is_initialized() {
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_realtime_enter() {
-  __rtsan::GetContextForThisThread().RealtimePush();
+  GetContextForThisThread().RealtimePush();
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_realtime_exit() {
-  __rtsan::GetContextForThisThread().RealtimePop();
+  GetContextForThisThread().RealtimePop();
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_disable() {
-  __rtsan::GetContextForThisThread().BypassPush();
+  GetContextForThisThread().BypassPush();
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void __rtsan_enable() {
-  __rtsan::GetContextForThisThread().BypassPop();
+  GetContextForThisThread().BypassPop();
 }
 
 SANITIZER_INTERFACE_ATTRIBUTE void
diff --git a/compiler-rt/lib/rtsan/rtsan_context.cpp b/compiler-rt/lib/rtsan/rtsan_context.cpp
index 1cf1791f0aaf8..536d62e81e2fb 100644
--- a/compiler-rt/lib/rtsan/rtsan_context.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_context.cpp
@@ -17,6 +17,7 @@
 #include <pthread.h>
 
 using namespace __sanitizer;
+using namespace __rtsan;
 
 static pthread_key_t context_key;
 static pthread_once_t key_once = PTHREAD_ONCE_INIT;
@@ -31,12 +32,12 @@ static __rtsan::Context &GetContextForThisThreadImpl() {
   };
 
   pthread_once(&key_once, MakeThreadLocalContextKey);
-  __rtsan::Context *current_thread_context =
-      static_cast<__rtsan::Context *>(pthread_getspecific(context_key));
+  Context *current_thread_context =
+      static_cast<Context *>(pthread_getspecific(context_key));
   if (current_thread_context == nullptr) {
-    current_thread_context = static_cast<__rtsan::Context *>(
-        __sanitizer::InternalAlloc(sizeof(__rtsan::Context)));
-    new (current_thread_context) __rtsan::Context();
+    current_thread_context =
+        static_cast<Context *>(InternalAlloc(sizeof(Context)));
+    new (current_thread_context) Context();
     pthread_setspecific(context_key, current_thread_context);
   }
 
@@ -57,6 +58,6 @@ bool __rtsan::Context::InRealtimeContext() const { return realtime_depth_ > 0; }
 
 bool __rtsan::Context::IsBypassed() const { return bypass_depth_ > 0; }
 
-__rtsan::Context &__rtsan::GetContextForThisThread() {
+Context &__rtsan::GetContextForThisThread() {
   return GetContextForThisThreadImpl();
 }
diff --git a/compiler-rt/lib/rtsan/rtsan_diagnostics.cpp b/compiler-rt/lib/rtsan/rtsan_diagnostics.cpp
index f82001f5b2057..ecba30d2ab8df 100644
--- a/compiler-rt/lib/rtsan/rtsan_diagnostics.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_diagnostics.cpp
@@ -31,7 +31,7 @@ void BufferedStackTrace::UnwindImpl(uptr pc, uptr bp, void *context,
 } // namespace __sanitizer
 
 namespace {
-class Decorator : public __sanitizer::SanitizerCommonDecorator {
+class Decorator : public SanitizerCommonDecorator {
 public:
   Decorator() : SanitizerCommonDecorator() {}
   const char *FunctionName() const { return Green(); }
@@ -39,13 +39,6 @@ class Decorator : public __sanitizer::SanitizerCommonDecorator {
 };
 } // namespace
 
-static void PrintStackTrace(uptr pc, uptr bp) {
-  BufferedStackTrace stack{};
-
-  stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal);
-  stack.Print();
-}
-
 static void PrintError(const Decorator &decorator,
                        const DiagnosticsInfo &info) {
   const auto ErrorTypeStr = [&info]() -> const char * {
@@ -91,5 +84,4 @@ void __rtsan::PrintDiagnostics(const DiagnosticsInfo &info) {
   PrintError(d, info);
   PrintReason(d, info);
   Printf("%s", d.Default());
-  PrintStackTrace(info.pc, info.bp);
 }
diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
index 9cc7214aef85c..63b0ca28a1f40 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
@@ -157,22 +157,38 @@ INTERCEPTOR(int, creat64, const char *path, mode_t mode) {
 INTERCEPTOR(int, fcntl, int filedes, int cmd, ...) {
   __rtsan_notify_intercepted_call("fcntl");
 
-  va_list args;
-  va_start(args, cmd);
-
   // Following precedent here. The linux source (fcntl.c, do_fcntl) accepts the
   // final argument in a variable that will hold the largest of the possible
-  // argument types (pointers and ints are typical in fcntl) It is then assumed
-  // that the implementation of fcntl will cast it properly depending on cmd.
+  // argument types. It is then assumed that the implementation of fcntl will
+  // cast it properly depending on cmd.
   //
-  // This is also similar to what is done in
-  // sanitizer_common/sanitizer_common_syscalls.inc
-  const unsigned long arg = va_arg(args, unsigned long);
-  int result = REAL(fcntl)(filedes, cmd, arg);
+  // The two types we expect for possible args are `struct flock*` and `int`
+  // we will cast to `intptr_t` which should hold both comfortably.
+  // Why `intptr_t`? It should fit both types, and it follows the freeBSD
+  // approach linked below.
+  using arg_type = intptr_t;
+  static_assert(sizeof(arg_type) >= sizeof(struct flock *));
+  static_assert(sizeof(arg_type) >= sizeof(int));
+
+  // Some cmds will not actually have an argument passed in this va_list.
+  // Calling va_arg when no arg exists is UB, however all currently
+  // supported architectures will give us a result in all three cases
+  // (no arg/int arg/struct flock* arg)
+  // va_arg() will generally read the next argument register or the
+  // stack. If we ever support an arch like CHERI with bounds checking, we
+  // may have to re-evaluate this approach.
+  //
+  // More discussion, and other examples following this approach
+  // https://discourse.llvm.org/t/how-to-write-an-interceptor-for-fcntl/81203
+  // https://reviews.freebsd.org/D46403
+  // https://github.com/bminor/glibc/blob/c444cc1d8335243c5c4e636d6a26c472df85522c/sysdeps/unix/sysv/linux/fcntl64.c#L37-L46
 
+  va_list args;
+  va_start(args, cmd);
+  const arg_type arg = va_arg(args, arg_type);
   va_end(args);
 
-  return result;
+  return REAL(fcntl)(filedes, cmd, arg);
 }
 
 #if SANITIZER_INTERCEPT_FCNTL64
diff --git a/compiler-rt/lib/rtsan/rtsan_stats.cpp b/compiler-rt/lib/rtsan/rtsan_stats.cpp
index 7c1ccf2876f08..dac7b23c3ef52 100644
--- a/compiler-rt/lib/rtsan/rtsan_stats.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_stats.cpp
@@ -19,17 +19,27 @@ using namespace __sanitizer;
 using namespace __rtsan;
 
 static atomic_uint32_t rtsan_total_error_count{0};
+static atomic_uint32_t rtsan_unique_error_count{0};
 
 void __rtsan::IncrementTotalErrorCount() {
   atomic_fetch_add(&rtsan_total_error_count, 1, memory_order_relaxed);
 }
 
+void __rtsan::IncrementUniqueErrorCount() {
+  atomic_fetch_add(&rtsan_unique_error_count, 1, memory_order_relaxed);
+}
+
 static u32 GetTotalErrorCount() {
   return atomic_load(&rtsan_total_error_count, memory_order_relaxed);
 }
 
+static u32 GetUniqueErrorCount() {
+  return atomic_load(&rtsan_unique_error_count, memory_order_relaxed);
+}
+
 void __rtsan::PrintStatisticsSummary() {
   ScopedErrorReportLock l;
   Printf("RealtimeSanitizer exit stats:\n");
   Printf("    Total error count: %u\n", GetTotalErrorCount());
+  Printf("    Unique error count: %u\n", GetUniqueErrorCount());
 }
diff --git a/compiler-rt/lib/rtsan/rtsan_stats.h b/compiler-rt/lib/rtsan/rtsan_stats.h
index 3aa30f6a5db76..a72098792c89c 100644
--- a/compiler-rt/lib/rtsan/rtsan_stats.h
+++ b/compiler-rt/lib/rtsan/rtsan_stats.h
@@ -15,6 +15,7 @@
 namespace __rtsan {
 
 void IncrementTotalErrorCount();
+void IncrementUniqueErrorCount();
 
 void PrintStatisticsSummary();
 
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_assertions.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_assertions.cpp
index 58f7dbae96e9f..3b279989a49cb 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_assertions.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_assertions.cpp
@@ -23,7 +23,7 @@ class TestRtsanAssertions : public ::testing::Test {
   void SetUp() override { __rtsan_ensure_initialized(); }
 };
 
-static void ExpectViolationAction(__rtsan::Context &context,
+static void ExpectViolationAction(Context &context,
                                   bool expect_violation_callback) {
   ::testing::MockFunction<void()> mock_on_violation;
   EXPECT_CALL(mock_on_violation, Call).Times(expect_violation_callback ? 1 : 0);
@@ -32,14 +32,14 @@ static void ExpectViolationAction(__rtsan::Context &context,
 
 TEST_F(TestRtsanAssertions,
        ExpectNotRealtimeDoesNotCallViolationActionIfNotInRealtimeContext) {
-  __rtsan::Context context{};
+  Context context{};
   ASSERT_FALSE(context.InRealtimeContext());
   ExpectViolationAction(context, false);
 }
 
 TEST_F(TestRtsanAssertions,
        ExpectNotRealtimeCallsViolationActionIfInRealtimeContext) {
-  __rtsan::Context context{};
+  Context context{};
   context.RealtimePush();
   ASSERT_TRUE(context.InRealtimeContext());
   ExpectViolationAction(context, true);
@@ -47,7 +47,7 @@ TEST_F(TestRtsanAssertions,
 
 TEST_F(TestRtsanAssertions,
        ExpectNotRealtimeDoesNotCallViolationActionIfRealtimeButBypassed) {
-  __rtsan::Context context{};
+  Context context{};
   context.RealtimePush();
   context.BypassPush();
   ASSERT_TRUE(context.IsBypassed());
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_context.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_context.cpp
index 7551f67b38d78..2b6f53b4f572d 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_context.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_context.cpp
@@ -15,6 +15,7 @@
 
 #include <gtest/gtest.h>
 
+using namespace __rtsan;
 using namespace ::testing;
 
 class TestRtsanContext : public Test {
@@ -23,18 +24,18 @@ class TestRtsanContext : public Test {
 };
 
 TEST_F(TestRtsanContext, IsNotRealtimeAfterDefaultConstruction) {
-  __rtsan::Context context{};
+  Context context{};
   EXPECT_THAT(context.InRealtimeContext(), Eq(false));
 }
 
 TEST_F(TestRtsanContext, IsRealtimeAfterRealtimePush) {
-  __rtsan::Context context{};
+  Context context{};
   context.RealtimePush();
   EXPECT_THAT(context.InRealtimeContext(), Eq(true));
 }
 
 TEST_F(TestRtsanContext, IsNotRealtimeAfterRealtimePushAndPop) {
-  __rtsan::Context context{};
+  Context context{};
   context.RealtimePush();
   ASSERT_THAT(context.InRealtimeContext(), Eq(true));
   context.RealtimePop();
@@ -42,7 +43,7 @@ TEST_F(TestRtsanContext, IsNotRealtimeAfterRealtimePushAndPop) {
 }
 
 TEST_F(TestRtsanContext, RealtimeContextStateIsStatefullyTracked) {
-  __rtsan::Context context{};
+  Context context{};
   auto const ExpectRealtime = [&context](bool is_rt) {
     EXPECT_THAT(context.InRealtimeContext(), Eq(is_rt));
   };
@@ -64,18 +65,18 @@ TEST_F(TestRtsanContext, RealtimeContextStateIsStatefullyTracked) {
 }
 
 TEST_F(TestRtsanContext, IsNotBypassedAfterDefaultConstruction) {
-  __rtsan::Context context{};
+  Context context{};
   EXPECT_THAT(context.IsBypassed(), Eq(false));
 }
 
 TEST_F(TestRtsanContext, IsBypassedAfterBypassPush) {
-  __rtsan::Context context{};
+  Context context{};
   context.BypassPush();
   EXPECT_THAT(context.IsBypassed(), Eq(true));
 }
 
 TEST_F(TestRtsanContext, BypassedStateIsStatefullyTracked) {
-  __rtsan::Context context{};
+  Context context{};
   auto const ExpectBypassed = [&context](bool is_bypassed) {
     EXPECT_THAT(context.IsBypassed(), Eq(is_bypassed));
   };
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 4fc99197aae3d..525bc1038619d 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -217,27 +217,14 @@ static void GetGLibcVersion(int *major, int *minor, int *patch) {
   *minor = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
   *patch = (*p == '.') ? internal_simple_strtoll(p + 1, &p, 10) : 0;
 }
-#  endif  // SANITIZER_GLIBC && !SANITIZER_GO
 
-// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
-// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
-// to get the pointer to thread-specific data keys in the thread control block.
-#  if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
-// sizeof(struct pthread) from glibc.
-static uptr thread_descriptor_size;
-
-// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.
 static uptr ThreadDescriptorSizeFallback() {
 #    if defined(__x86_64__) || defined(__i386__) || defined(__arm__) || \
         SANITIZER_RISCV64
-#      if SANITIZER_GLIBC
   int major;
   int minor;
   int patch;
   GetGLibcVersion(&major, &minor, &patch);
-#      else   // SANITIZER_GLIBC
-  return 0;
-#      endif  // SANITIZER_GLIBC
 #    endif
 
 #    if defined(__x86_64__) || defined(__i386__) || defined(__arm__)
@@ -304,6 +291,48 @@ static uptr ThreadDescriptorSizeFallback() {
   return 1776;  // from glibc.ppc64le 2.20-8.fc21
 #    endif
 }
+#  endif  // SANITIZER_GLIBC && !SANITIZER_GO
+
+#  if SANITIZER_FREEBSD && !SANITIZER_GO
+// FIXME: Implementation is very GLIBC specific, but it's used by FreeBSD.
+static uptr ThreadDescriptorSizeFallback() {
+#    if defined(__s390__) || defined(__sparc__)
+  // The size of a prefix of TCB including pthread::{specific_1stblock,specific}
+  // suffices. Just return offsetof(struct pthread, specific_used), which hasn't
+  // changed since 2007-05. Technically this applies to i386/x86_64 as well but
+  // we call _dl_get_tls_static_info and need the precise size of struct
+  // pthread.
+  return FIRST_32_SECOND_64(524, 1552);
+#    endif
+
+#    if defined(__mips__)
+  // TODO(sagarthakur): add more values as per different glibc versions.
+  return FIRST_32_SECOND_64(1152, 1776);
+#    endif
+
+#    if SANITIZER_LOONGARCH64
+  return 1856;  // from glibc 2.36
+#    endif
+
+#    if defined(__aarch64__)
+  // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22.
+  return 1776;
+#    endif
+
+#    if defined(__powerpc64__)
+  return 1776;  // from glibc.ppc64le 2.20-8.fc21
+#    endif
+
+  return 0;
+}
+#  endif  // SANITIZER_FREEBSD && !SANITIZER_GO
+
+#  if (SANITIZER_FREEBSD || SANITIZER_GLIBC) && !SANITIZER_GO
+// On glibc x86_64, ThreadDescriptorSize() needs to be precise due to the usage
+// of g_tls_size. On other targets, ThreadDescriptorSize() is only used by lsan
+// to get the pointer to thread-specific data keys in the thread control block.
+// sizeof(struct pthread) from glibc.
+static uptr thread_descriptor_size;
 
 uptr ThreadDescriptorSize() { return thread_descriptor_size; }
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
index b44e016a0e5bc..5ff8d1832556f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
@@ -433,7 +433,9 @@ void MemoryMappingLayout::DumpListOfModules(
   MemoryMappedSegmentData data;
   segment.data_ = &data;
   while (Next(&segment)) {
-    if (segment.filename[0] == '\0') continue;
+    // skip the __PAGEZERO segment, its vmsize is 0
+    if (segment.filename[0] == '\0' || (segment.start == segment.end))
+      continue;
     LoadedModule *cur_module = nullptr;
     if (!modules->empty() &&
         0 == internal_strcmp(segment.filename, modules->back().full_name())) {
diff --git a/compiler-rt/test/rtsan/deduplicate_errors.cpp b/compiler-rt/test/rtsan/deduplicate_errors.cpp
new file mode 100644
index 0000000000000..7d60d4d7da7dd
--- /dev/null
+++ b/compiler-rt/test/rtsan/deduplicate_errors.cpp
@@ -0,0 +1,39 @@
+// RUN: %clangxx -fsanitize=realtime %s -o %t
+// RUN: env RTSAN_OPTIONS="halt_on_error=false,print_stats_on_exit=true" %run %t 2>&1 | FileCheck %s
+
+// UNSUPPORTED: ios
+
+// Intent: Ensure all errors are deduplicated.
+
+#include <unistd.h>
+
+const int kNumViolations = 10;
+
+void violation() [[clang::nonblocking]] {
+  for (int i = 0; i < kNumViolations; i++)
+    usleep(1);
+}
+
+void violation2() [[clang::nonblocking]] {
+  for (int i = 0; i < kNumViolations; i++)
+    violation();
+}
+
+void double_violation() [[clang::nonblocking]] {
+  violation();
+  violation2();
+}
+
+int main() {
+  violation();        // 1 unique errors here, but 10 total
+  violation2();       // 1 unique errors here, but 100 total
+  double_violation(); // 2 unique errors here, but 110 total
+  return 0;
+}
+
+// CHECK-COUNT-4: ==ERROR:
+// CHECK-NOT: ==ERROR:
+
+// CHECK: RealtimeSanitizer exit stats:
+// CHECK-NEXT: Total error count: 220
+// CHECK-NEXT: Unique error count: 4
diff --git a/compiler-rt/test/rtsan/exit_stats.cpp b/compiler-rt/test/rtsan/exit_stats.cpp
index b46a0fd62bac1..4341fbb0f9cf2 100644
--- a/compiler-rt/test/rtsan/exit_stats.cpp
+++ b/compiler-rt/test/rtsan/exit_stats.cpp
@@ -21,3 +21,4 @@ int main() {
 
 // CHECK: RealtimeSanitizer exit stats:
 // CHECK-NEXT: Total error count: 10
+// CHECK-NEXT: Unique error count: 1
diff --git a/flang/include/flang/Frontend/LangOptions.def b/flang/include/flang/Common/LangOptions.def
similarity index 100%
rename from flang/include/flang/Frontend/LangOptions.def
rename to flang/include/flang/Common/LangOptions.def
diff --git a/flang/include/flang/Frontend/LangOptions.h b/flang/include/flang/Common/LangOptions.h
similarity index 79%
rename from flang/include/flang/Frontend/LangOptions.h
rename to flang/include/flang/Common/LangOptions.h
index 57d86d46df5ab..52a45047deb0e 100644
--- a/flang/include/flang/Frontend/LangOptions.h
+++ b/flang/include/flang/Common/LangOptions.h
@@ -12,15 +12,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef FORTRAN_FRONTEND_LANGOPTIONS_H
-#define FORTRAN_FRONTEND_LANGOPTIONS_H
+#ifndef FORTRAN_COMMON_LANGOPTIONS_H
+#define FORTRAN_COMMON_LANGOPTIONS_H
 
 #include <string>
 #include <vector>
 
 #include "llvm/TargetParser/Triple.h"
 
-namespace Fortran::frontend {
+namespace Fortran::common {
 
 /// Bitfields of LangOptions, split out from LangOptions to ensure
 /// that this large collection of bitfields is a trivial class type.
@@ -37,12 +37,12 @@ class LangOptionsBase {
 
 #define LANGOPT(Name, Bits, Default) unsigned Name : Bits;
 #define ENUM_LANGOPT(Name, Type, Bits, Default)
-#include "flang/Frontend/LangOptions.def"
+#include "flang/Common/LangOptions.def"
 
 protected:
 #define LANGOPT(Name, Bits, Default)
 #define ENUM_LANGOPT(Name, Type, Bits, Default) unsigned Name : Bits;
-#include "flang/Frontend/LangOptions.def"
+#include "flang/Common/LangOptions.def"
 };
 
 /// Tracks various options which control the dialect of Fortran that is
@@ -52,10 +52,10 @@ class LangOptions : public LangOptionsBase {
 public:
   // Define accessors/mutators for code generation options of enumeration type.
 #define LANGOPT(Name, Bits, Default)
-#define ENUM_LANGOPT(Name, Type, Bits, Default)                                \
-  Type get##Name() const { return static_cast<Type>(Name); }                   \
+#define ENUM_LANGOPT(Name, Type, Bits, Default) \
+  Type get##Name() const { return static_cast<Type>(Name); } \
   void set##Name(Type Value) { Name = static_cast<unsigned>(Value); }
-#include "flang/Frontend/LangOptions.def"
+#include "flang/Common/LangOptions.def"
 
   /// Name of the IR file that contains the result of the OpenMP target
   /// host code generation.
@@ -67,6 +67,6 @@ class LangOptions : public LangOptionsBase {
   LangOptions();
 };
 
-} // end namespace Fortran::frontend
+} // end namespace Fortran::common
 
-#endif // FORTRAN_FRONTEND_LANGOPTIONS_H
+#endif // FORTRAN_COMMON_LANGOPTIONS_H
diff --git a/flang/include/flang/Frontend/CompilerInvocation.h b/flang/include/flang/Frontend/CompilerInvocation.h
index d1646f585cf85..50d908d083202 100644
--- a/flang/include/flang/Frontend/CompilerInvocation.h
+++ b/flang/include/flang/Frontend/CompilerInvocation.h
@@ -13,9 +13,9 @@
 #ifndef FORTRAN_FRONTEND_COMPILERINVOCATION_H
 #define FORTRAN_FRONTEND_COMPILERINVOCATION_H
 
+#include "flang/Common/LangOptions.h"
 #include "flang/Frontend/CodeGenOptions.h"
 #include "flang/Frontend/FrontendOptions.h"
-#include "flang/Frontend/LangOptions.h"
 #include "flang/Frontend/PreprocessorOptions.h"
 #include "flang/Frontend/TargetOptions.h"
 #include "flang/Lower/LoweringOptions.h"
@@ -84,7 +84,7 @@ class CompilerInvocation : public CompilerInvocationBase {
   Fortran::frontend::CodeGenOptions codeGenOpts;
 
   /// Options controlling language dialect.
-  Fortran::frontend::LangOptions langOpts;
+  Fortran::common::LangOptions langOpts;
 
   // The original invocation of the compiler driver.
   // This string will be set as the return value from the COMPILER_OPTIONS
@@ -158,8 +158,8 @@ class CompilerInvocation : public CompilerInvocationBase {
   CodeGenOptions &getCodeGenOpts() { return codeGenOpts; }
   const CodeGenOptions &getCodeGenOpts() const { return codeGenOpts; }
 
-  LangOptions &getLangOpts() { return langOpts; }
-  const LangOptions &getLangOpts() const { return langOpts; }
+  Fortran::common::LangOptions &getLangOpts() { return langOpts; }
+  const Fortran::common::LangOptions &getLangOpts() const { return langOpts; }
 
   Fortran::lower::LoweringOptions &getLoweringOpts() { return loweringOpts; }
   const Fortran::lower::LoweringOptions &getLoweringOpts() const {
diff --git a/flang/include/flang/Lower/CallInterface.h b/flang/include/flang/Lower/CallInterface.h
index 1fb390455733f..72bc9dd890a94 100644
--- a/flang/include/flang/Lower/CallInterface.h
+++ b/flang/include/flang/Lower/CallInterface.h
@@ -248,6 +248,7 @@ class CallInterface {
   CallInterface(Fortran::lower::AbstractConverter &c) : converter{c} {}
   /// CRTP handle.
   T &side() { return *static_cast<T *>(this); }
+  const T &side() const { return *static_cast<const T *>(this); }
   /// Entry point to be called by child ctor to analyze the signature and
   /// create/find the mlir::func::FuncOp. Child needs to be initialized first.
   void declare();
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 50e18792a167a..cdbefdb234148 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -160,6 +160,10 @@ static constexpr llvm::StringRef getFuncRecursiveAttrName() {
   return "fir.func_recursive";
 }
 
+static constexpr llvm::StringRef getFortranProcedureFlagsAttrName() {
+  return "fir.proc_attrs";
+}
+
 // Attribute for an alloca that is a trivial adaptor for converting a value to
 // pass-by-ref semantics for a VALUE parameter. The optimizer may be able to
 // eliminate these.
diff --git a/flang/include/flang/Semantics/semantics.h b/flang/include/flang/Semantics/semantics.h
index e73f9d2e85d58..2a326074b3dc6 100644
--- a/flang/include/flang/Semantics/semantics.h
+++ b/flang/include/flang/Semantics/semantics.h
@@ -12,6 +12,7 @@
 #include "scope.h"
 #include "symbol.h"
 #include "flang/Common/Fortran-features.h"
+#include "flang/Common/LangOptions.h"
 #include "flang/Evaluate/common.h"
 #include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/target.h"
@@ -65,7 +66,8 @@ using ConstructStack = std::vector<ConstructNode>;
 class SemanticsContext {
 public:
   SemanticsContext(const common::IntrinsicTypeDefaultKinds &,
-      const common::LanguageFeatureControl &, parser::AllCookedSources &);
+      const common::LanguageFeatureControl &, const common::LangOptions &,
+      parser::AllCookedSources &);
   ~SemanticsContext();
 
   const common::IntrinsicTypeDefaultKinds &defaultKinds() const {
@@ -73,7 +75,8 @@ class SemanticsContext {
   }
   const common::LanguageFeatureControl &languageFeatures() const {
     return languageFeatures_;
-  };
+  }
+  const common::LangOptions &langOptions() const { return langOpts_; }
   int GetDefaultKind(TypeCategory) const;
   int doublePrecisionKind() const {
     return defaultKinds_.doublePrecisionKind();
@@ -273,6 +276,7 @@ class SemanticsContext {
 
   const common::IntrinsicTypeDefaultKinds &defaultKinds_;
   const common::LanguageFeatureControl &languageFeatures_;
+  const common::LangOptions &langOpts_;
   parser::AllCookedSources &allCookedSources_;
   std::optional<parser::CharBlock> location_;
   std::vector<std::string> searchDirectories_;
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index 75fd783af237d..3e703de545950 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -13,9 +13,9 @@
 #ifndef FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 #define FORTRAN_TOOLS_CROSS_TOOL_HELPERS_H
 
+#include "flang/Common/LangOptions.h"
 #include "flang/Common/MathOptionsBase.h"
 #include "flang/Frontend/CodeGenOptions.h"
-#include "flang/Frontend/LangOptions.h"
 #include <cstdint>
 
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
@@ -145,7 +145,7 @@ struct OffloadModuleOpts {
         OMPTargetTriples(OMPTargetTriples.begin(), OMPTargetTriples.end()),
         NoGPULib(NoGPULib) {}
 
-  OffloadModuleOpts(Fortran::frontend::LangOptions &Opts)
+  OffloadModuleOpts(Fortran::common::LangOptions &Opts)
       : OpenMPTargetDebug(Opts.OpenMPTargetDebug),
         OpenMPTeamSubscription(Opts.OpenMPTeamSubscription),
         OpenMPThreadSubscription(Opts.OpenMPThreadSubscription),
diff --git a/flang/lib/Common/CMakeLists.txt b/flang/lib/Common/CMakeLists.txt
index 6579e9cdf8249..be72391847f3d 100644
--- a/flang/lib/Common/CMakeLists.txt
+++ b/flang/lib/Common/CMakeLists.txt
@@ -39,6 +39,7 @@ add_flang_library(FortranCommon
   Fortran-features.cpp
   default-kinds.cpp
   idioms.cpp
+  LangOptions.cpp
   Version.cpp
   ${version_inc}
 
diff --git a/flang/lib/Frontend/LangOptions.cpp b/flang/lib/Common/LangOptions.cpp
similarity index 82%
rename from flang/lib/Frontend/LangOptions.cpp
rename to flang/lib/Common/LangOptions.cpp
index a08cb363384c6..415c715156e7b 100644
--- a/flang/lib/Frontend/LangOptions.cpp
+++ b/flang/lib/Common/LangOptions.cpp
@@ -10,15 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "flang/Frontend/LangOptions.h"
+#include "flang/Common/LangOptions.h"
 #include <string.h>
 
-namespace Fortran::frontend {
+namespace Fortran::common {
 
 LangOptions::LangOptions() {
 #define LANGOPT(Name, Bits, Default) Name = Default;
 #define ENUM_LANGOPT(Name, Type, Bits, Default) set##Name(Default);
-#include "flang/Frontend/LangOptions.def"
+#include "flang/Common/LangOptions.def"
 }
 
-} // end namespace Fortran::frontend
+} // end namespace Fortran::common
diff --git a/flang/lib/Frontend/CMakeLists.txt b/flang/lib/Frontend/CMakeLists.txt
index ebfdb14b534bb..e954800c3b88b 100644
--- a/flang/lib/Frontend/CMakeLists.txt
+++ b/flang/lib/Frontend/CMakeLists.txt
@@ -8,7 +8,6 @@ add_flang_library(flangFrontend
   FrontendAction.cpp
   FrontendActions.cpp
   FrontendOptions.cpp
-  LangOptions.cpp
   TextDiagnosticPrinter.cpp
   TextDiagnosticBuffer.cpp
   TextDiagnostic.cpp
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 90c327546198b..d68534d5509fe 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -40,6 +40,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
 #include "llvm/TargetParser/Triple.h"
+#include <algorithm>
 #include <cstdlib>
 #include <memory>
 #include <optional>
@@ -830,14 +831,20 @@ static bool parseSemaArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
   unsigned numErrorsBefore = diags.getNumErrors();
 
   // -J/module-dir option
-  auto moduleDirList =
+  std::vector<std::string> moduleDirList =
       args.getAllArgValues(clang::driver::options::OPT_module_dir);
-  // User can only specify -J/-module-dir once
+  // User can only specify one -J/-module-dir directory, but may repeat
+  // -J/-module-dir as long as the directory is the same each time.
   // https://gcc.gnu.org/onlinedocs/gfortran/Directory-Options.html
+  std::sort(moduleDirList.begin(), moduleDirList.end());
+  moduleDirList.erase(std::unique(moduleDirList.begin(), moduleDirList.end()),
+                      moduleDirList.end());
   if (moduleDirList.size() > 1) {
     const unsigned diagID =
         diags.getCustomDiagID(clang::DiagnosticsEngine::Error,
-                              "Only one '-module-dir/-J' option allowed");
+                              "Only one '-module-dir/-J' directory allowed. "
+                              "'-module-dir/-J' may be given multiple times "
+                              "but the directory must be the same each time.");
     diags.Report(diagID);
   }
   if (moduleDirList.size() == 1)
@@ -1110,17 +1117,17 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
 static bool parseFloatingPointArgs(CompilerInvocation &invoc,
                                    llvm::opt::ArgList &args,
                                    clang::DiagnosticsEngine &diags) {
-  LangOptions &opts = invoc.getLangOpts();
+  Fortran::common::LangOptions &opts = invoc.getLangOpts();
 
   if (const llvm::opt::Arg *a =
           args.getLastArg(clang::driver::options::OPT_ffp_contract)) {
     const llvm::StringRef val = a->getValue();
-    enum LangOptions::FPModeKind fpContractMode;
+    enum Fortran::common::LangOptions::FPModeKind fpContractMode;
 
     if (val == "off")
-      fpContractMode = LangOptions::FPM_Off;
+      fpContractMode = Fortran::common::LangOptions::FPM_Off;
     else if (val == "fast")
-      fpContractMode = LangOptions::FPM_Fast;
+      fpContractMode = Fortran::common::LangOptions::FPM_Fast;
     else {
       diags.Report(clang::diag::err_drv_unsupported_option_argument)
           << a->getSpelling() << val;
@@ -1161,7 +1168,7 @@ static bool parseFloatingPointArgs(CompilerInvocation &invoc,
     opts.ReciprocalMath = true;
     opts.ApproxFunc = true;
     opts.NoSignedZeros = true;
-    opts.setFPContractMode(LangOptions::FPM_Fast);
+    opts.setFPContractMode(Fortran::common::LangOptions::FPM_Fast);
   }
 
   return true;
@@ -1194,7 +1201,7 @@ static bool parseVScaleArgs(CompilerInvocation &invoc, llvm::opt::ArgList &args,
     return false;
   }
 
-  LangOptions &opts = invoc.getLangOpts();
+  Fortran::common::LangOptions &opts = invoc.getLangOpts();
   if (vscaleMin) {
     llvm::StringRef argValue = llvm::StringRef(vscaleMin->getValue());
     unsigned vscaleMinVal;
@@ -1531,7 +1538,8 @@ CompilerInvocation::getSemanticsCtx(
   auto &fortranOptions = getFortranOpts();
 
   auto semanticsContext = std::make_unique<semantics::SemanticsContext>(
-      getDefaultKinds(), fortranOptions.features, allCookedSources);
+      getDefaultKinds(), fortranOptions.features, getLangOpts(),
+      allCookedSources);
 
   semanticsContext->set_moduleDirectory(getModuleDir())
       .set_searchDirectories(fortranOptions.searchDirectories)
@@ -1556,14 +1564,14 @@ void CompilerInvocation::setLoweringOptions() {
   loweringOpts.setOptimizeTranspose(codegenOpts.OptimizationLevel > 0);
   loweringOpts.setUnderscoring(codegenOpts.Underscoring);
 
-  const LangOptions &langOptions = getLangOpts();
+  const Fortran::common::LangOptions &langOptions = getLangOpts();
   Fortran::common::MathOptionsBase &mathOpts = loweringOpts.getMathOptions();
   // TODO: when LangOptions are finalized, we can represent
   //       the math related options using Fortran::commmon::MathOptionsBase,
   //       so that we can just copy it into LoweringOptions.
   mathOpts
       .setFPContractEnabled(langOptions.getFPContractMode() ==
-                            LangOptions::FPM_Fast)
+                            Fortran::common::LangOptions::FPM_Fast)
       .setNoHonorInfs(langOptions.NoHonorInfs)
       .setNoHonorNaNs(langOptions.NoHonorNaNs)
       .setApproxFunc(langOptions.ApproxFunc)
diff --git a/flang/lib/Frontend/TextDiagnosticPrinter.cpp b/flang/lib/Frontend/TextDiagnosticPrinter.cpp
index dc182d68a1a97..2ab02c0b70ab5 100644
--- a/flang/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/flang/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -38,8 +38,8 @@ TextDiagnosticPrinter::~TextDiagnosticPrinter() {}
 static void printRemarkOption(llvm::raw_ostream &os,
                               clang::DiagnosticsEngine::Level level,
                               const clang::Diagnostic &info) {
-  llvm::StringRef opt = info.getDiags()->getDiagnosticIDs()
-      ->getWarningOptionForDiag(info.getID());
+  llvm::StringRef opt =
+      clang::DiagnosticIDs::getWarningOptionForDiag(info.getID());
   if (!opt.empty()) {
     // We still need to check if the level is a Remark since, an unknown option
     // warning could be printed i.e. [-Wunknown-warning-option]
diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index f541f84738291..7fc6b14f9c660 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -582,6 +582,7 @@ mlir::Value Fortran::lower::CalleeInterface::getHostAssociatedTuple() const {
 
 static void addSymbolAttribute(mlir::func::FuncOp func,
                                const Fortran::semantics::Symbol &sym,
+                               fir::FortranProcedureFlagsEnumAttr procAttrs,
                                mlir::MLIRContext &mlirContext) {
   const Fortran::semantics::Symbol &ultimate = sym.GetUltimate();
   // The link between an internal procedure and its host procedure is lost
@@ -611,16 +612,8 @@ static void addSymbolAttribute(mlir::func::FuncOp func,
     }
   }
 
-  // Set procedure attributes to the func op.
-  if (IsPureProcedure(sym))
-    func->setAttr(fir::getFuncPureAttrName(),
-                  mlir::UnitAttr::get(&mlirContext));
-  if (IsElementalProcedure(sym))
-    func->setAttr(fir::getFuncElementalAttrName(),
-                  mlir::UnitAttr::get(&mlirContext));
-  if (sym.attrs().test(Fortran::semantics::Attr::RECURSIVE))
-    func->setAttr(fir::getFuncRecursiveAttrName(),
-                  mlir::UnitAttr::get(&mlirContext));
+  if (procAttrs)
+    func->setAttr(fir::getFortranProcedureFlagsAttrName(), procAttrs);
 
   // Only add this on bind(C) functions for which the symbol is not reflected in
   // the current context.
@@ -703,6 +696,7 @@ void Fortran::lower::CallInterface<T>::declare() {
     func = fir::FirOpBuilder::getNamedFunction(module, symbolTable, name);
     if (!func) {
       mlir::Location loc = side().getCalleeLocation();
+      mlir::MLIRContext &mlirContext = converter.getMLIRContext();
       mlir::FunctionType ty = genFunctionType();
       func =
           fir::FirOpBuilder::createFunction(loc, module, name, ty, symbolTable);
@@ -712,7 +706,8 @@ void Fortran::lower::CallInterface<T>::declare() {
                         mlir::StringAttr::get(&converter.getMLIRContext(),
                                               sym->name().ToString()));
         } else {
-          addSymbolAttribute(func, *sym, converter.getMLIRContext());
+          addSymbolAttribute(func, *sym, getProcedureAttrs(&mlirContext),
+                             mlirContext);
         }
       }
       for (const auto &placeHolder : llvm::enumerate(inputs))
@@ -1550,8 +1545,8 @@ template <typename T>
 fir::FortranProcedureFlagsEnumAttr
 Fortran::lower::CallInterface<T>::getProcedureAttrs(
     mlir::MLIRContext *mlirContext) const {
+  fir::FortranProcedureFlagsEnum flags = fir::FortranProcedureFlagsEnum::none;
   if (characteristic) {
-    fir::FortranProcedureFlagsEnum flags = fir::FortranProcedureFlagsEnum::none;
     if (characteristic->IsBindC())
       flags = flags | fir::FortranProcedureFlagsEnum::bind_c;
     if (characteristic->IsPure())
@@ -1560,12 +1555,27 @@ Fortran::lower::CallInterface<T>::getProcedureAttrs(
       flags = flags | fir::FortranProcedureFlagsEnum::elemental;
     // TODO:
     // - SIMPLE: F2023, not yet handled by semantics.
-    // - NON_RECURSIVE: not part of the characteristics. Maybe this should
-    //   simply not be part of FortranProcedureFlagsEnum since cannot accurately
-    //   be known on the caller side.
-    if (flags != fir::FortranProcedureFlagsEnum::none)
-      return fir::FortranProcedureFlagsEnumAttr::get(mlirContext, flags);
   }
+
+  if constexpr (std::is_same_v<Fortran::lower::CalleeInterface, T>) {
+    // Only gather and set NON_RECURSIVE for procedure definition. It is
+    // meaningless on calls since this is not part of Fortran characteristics
+    // (Fortran 2023 15.3.1) so there is no way to always know if the procedure
+    // called is recursive or not.
+    if (const Fortran::semantics::Symbol *sym = side().getProcedureSymbol()) {
+      // Note: By default procedures are RECURSIVE unless
+      // -fno-automatic/-save/-Msave is set. NON_RECURSIVE is is made explicit
+      // in that case in FIR.
+      if (sym->attrs().test(Fortran::semantics::Attr::NON_RECURSIVE) ||
+          (sym->owner().context().languageFeatures().IsEnabled(
+               Fortran::common::LanguageFeature::DefaultSave) &&
+           !sym->attrs().test(Fortran::semantics::Attr::RECURSIVE))) {
+        flags = flags | fir::FortranProcedureFlagsEnum::non_recursive;
+      }
+    }
+  }
+  if (flags != fir::FortranProcedureFlagsEnum::none)
+    return fir::FortranProcedureFlagsEnumAttr::get(mlirContext, flags);
   return nullptr;
 }
 
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 960286732c90c..d528772f28724 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1681,7 +1681,6 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    mapTypes, deviceAddrSyms, deviceAddrLocs, deviceAddrTypes,
                    devicePtrSyms, devicePtrLocs, devicePtrTypes);
 
-  llvm::SmallVector<const semantics::Symbol *> privateSyms;
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
                            /*shouldCollectPreDeterminedSymbols=*/
                            lower::omp::isLastItemInQueue(item, queue),
@@ -1936,24 +1935,26 @@ static void genStandaloneDistribute(lower::AbstractConverter &converter,
   genDistributeClauses(converter, semaCtx, stmtCtx, item->clauses, loc,
                        distributeClauseOps);
 
-  // TODO: Support delayed privatization.
   DataSharingProcessor dsp(converter, semaCtx, item->clauses, eval,
                            /*shouldCollectPreDeterminedSymbols=*/true,
-                           /*useDelayedPrivatization=*/false, &symTable);
-  dsp.processStep1();
+                           enableDelayedPrivatizationStaging, &symTable);
+  dsp.processStep1(&distributeClauseOps);
+  llvm::SmallVector<mlir::Type> privateVarTypes{};
+
+  for (mlir::Value privateVar : distributeClauseOps.privateVars)
+    privateVarTypes.push_back(privateVar.getType());
 
   mlir::omp::LoopNestOperands loopNestClauseOps;
   llvm::SmallVector<const semantics::Symbol *> iv;
   genLoopNestClauses(converter, semaCtx, eval, item->clauses, loc,
                      loopNestClauseOps, iv);
 
-  // TODO: Populate entry block arguments with private variables.
   auto distributeOp = genWrapperOp<mlir::omp::DistributeOp>(
-      converter, loc, distributeClauseOps, /*blockArgTypes=*/{});
+      converter, loc, distributeClauseOps, privateVarTypes);
 
   genLoopNestOp(converter, symTable, semaCtx, eval, loc, queue, item,
-                loopNestClauseOps, iv,
-                /*wrapperSyms=*/{}, distributeOp.getRegion().getArguments(),
+                loopNestClauseOps, iv, dsp.getDelayedPrivSymbols(),
+                distributeOp.getRegion().getArguments(),
                 llvm::omp::Directive::OMPD_distribute, dsp);
 }
 
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 88293bcf36a78..efc28e9708e19 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -110,6 +110,26 @@ static unsigned getLenParamFieldId(mlir::Type ty) {
   return getTypeDescFieldId(ty) + 1;
 }
 
+static llvm::SmallVector<mlir::NamedAttribute>
+addLLVMOpBundleAttrs(mlir::ConversionPatternRewriter &rewriter,
+                     llvm::ArrayRef<mlir::NamedAttribute> attrs,
+                     int32_t numCallOperands) {
+  llvm::SmallVector<mlir::NamedAttribute> newAttrs;
+  newAttrs.reserve(attrs.size() + 2);
+
+  for (mlir::NamedAttribute attr : attrs) {
+    if (attr.getName() != "operandSegmentSizes")
+      newAttrs.push_back(attr);
+  }
+
+  newAttrs.push_back(rewriter.getNamedAttr(
+      "operandSegmentSizes",
+      rewriter.getDenseI32ArrayAttr({numCallOperands, 0})));
+  newAttrs.push_back(rewriter.getNamedAttr("op_bundle_sizes",
+                                           rewriter.getDenseI32ArrayAttr({})));
+  return newAttrs;
+}
+
 namespace {
 /// Lower `fir.address_of` operation to `llvm.address_of` operation.
 struct AddrOfOpConversion : public fir::FIROpConversion<fir::AddrOfOp> {
@@ -229,7 +249,8 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
         mlir::NamedAttribute attr = rewriter.getNamedAttr(
             "callee", mlir::SymbolRefAttr::get(memSizeFn));
         auto call = rewriter.create<mlir::LLVM::CallOp>(
-            loc, ity, lenParams, llvm::ArrayRef<mlir::NamedAttribute>{attr});
+            loc, ity, lenParams,
+            addLLVMOpBundleAttrs(rewriter, {attr}, lenParams.size()));
         size = call.getResult();
         llvmObjectType = ::getI8Type(alloc.getContext());
       } else {
@@ -559,7 +580,9 @@ struct CallOpConversion : public fir::FIROpConversion<fir::CallOp> {
     mlir::arith::AttrConvertFastMathToLLVM<fir::CallOp, mlir::LLVM::CallOp>
         attrConvert(call);
     rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        call, resultTys, adaptor.getOperands(), attrConvert.getAttrs());
+        call, resultTys, adaptor.getOperands(),
+        addLLVMOpBundleAttrs(rewriter, attrConvert.getAttrs(),
+                             adaptor.getOperands().size()));
     return mlir::success();
   }
 };
@@ -980,7 +1003,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
           loc, ity, size, integerCast(loc, rewriter, ity, opnd));
     heap->setAttr("callee", getMalloc(heap, rewriter));
     rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
-        heap, ::getLlvmPtrType(heap.getContext()), size, heap->getAttrs());
+        heap, ::getLlvmPtrType(heap.getContext()), size,
+        addLLVMOpBundleAttrs(rewriter, heap->getAttrs(), 1));
     return mlir::success();
   }
 
@@ -1037,9 +1061,9 @@ struct FreeMemOpConversion : public fir::FIROpConversion<fir::FreeMemOp> {
                   mlir::ConversionPatternRewriter &rewriter) const override {
     mlir::Location loc = freemem.getLoc();
     freemem->setAttr("callee", getFree(freemem, rewriter));
-    rewriter.create<mlir::LLVM::CallOp>(loc, mlir::TypeRange{},
-                                        mlir::ValueRange{adaptor.getHeapref()},
-                                        freemem->getAttrs());
+    rewriter.create<mlir::LLVM::CallOp>(
+        loc, mlir::TypeRange{}, mlir::ValueRange{adaptor.getHeapref()},
+        addLLVMOpBundleAttrs(rewriter, freemem->getAttrs(), 1));
     rewriter.eraseOp(freemem);
     return mlir::success();
   }
@@ -2671,7 +2695,8 @@ struct FieldIndexOpConversion : public fir::FIROpConversion<fir::FieldIndexOp> {
         "field", mlir::IntegerAttr::get(lowerTy().indexType(), index));
     rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
         field, lowerTy().offsetType(), adaptor.getOperands(),
-        llvm::ArrayRef<mlir::NamedAttribute>{callAttr, fieldAttr});
+        addLLVMOpBundleAttrs(rewriter, {callAttr, fieldAttr},
+                             adaptor.getOperands().size()));
     return mlir::success();
   }
 
diff --git a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
index 1390fae062b93..4aa14ca2c2bdd 100644
--- a/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
+++ b/flang/lib/Optimizer/Transforms/DebugTypeGenerator.cpp
@@ -295,23 +295,24 @@ mlir::LLVM::DITypeAttr DebugTypeGenerator::convertCharacterType(
     // variable that will contain that length. This variable is used as
     // 'stringLength' in DIStringTypeAttr.
     if (declOp && !declOp.getTypeparams().empty()) {
-      mlir::Operation *op = declOp.getTypeparams()[0].getDefiningOp();
-      if (auto unbox = mlir::dyn_cast_or_null<fir::UnboxCharOp>(op)) {
-        auto name =
-            mlir::StringAttr::get(context, "." + declOp.getUniqName().str());
-        mlir::OpBuilder builder(context);
-        builder.setInsertionPoint(declOp);
-        mlir::Type i64Ty = builder.getIntegerType(64);
-        auto convOp = builder.create<fir::ConvertOp>(unbox.getLoc(), i64Ty,
-                                                     unbox.getResult(1));
-        mlir::LLVM::DITypeAttr Ty = convertType(i64Ty, fileAttr, scope, declOp);
-        auto lvAttr = mlir::LLVM::DILocalVariableAttr::get(
-            context, scope, name, fileAttr, /*line=*/0, /*argNo=*/0,
-            /*alignInBits=*/0, Ty, mlir::LLVM::DIFlags::Artificial);
-        builder.create<mlir::LLVM::DbgValueOp>(convOp.getLoc(), convOp, lvAttr,
-                                               nullptr);
-        varAttr = mlir::cast<mlir::LLVM::DIVariableAttr>(lvAttr);
+      auto name =
+          mlir::StringAttr::get(context, "." + declOp.getUniqName().str());
+      mlir::OpBuilder builder(context);
+      builder.setInsertionPoint(declOp);
+      mlir::Value sizeVal = declOp.getTypeparams()[0];
+      mlir::Type type = sizeVal.getType();
+      if (!mlir::isa<mlir::IntegerType>(type) || !type.isSignlessInteger()) {
+        type = builder.getIntegerType(64);
+        sizeVal =
+            builder.create<fir::ConvertOp>(declOp.getLoc(), type, sizeVal);
       }
+      mlir::LLVM::DITypeAttr Ty = convertType(type, fileAttr, scope, declOp);
+      auto lvAttr = mlir::LLVM::DILocalVariableAttr::get(
+          context, scope, name, fileAttr, /*line=*/0, /*argNo=*/0,
+          /*alignInBits=*/0, Ty, mlir::LLVM::DIFlags::Artificial);
+      builder.create<mlir::LLVM::DbgValueOp>(declOp.getLoc(), sizeVal, lvAttr,
+                                             nullptr);
+      varAttr = mlir::cast<mlir::LLVM::DIVariableAttr>(lvAttr);
     }
   }
 
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index dfc3f3290a81b..2943ee5dd7552 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -16,25 +16,25 @@ namespace Fortran::semantics {
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
   void OmpStructureChecker::Enter(const parser::OmpClause::X &) { \
-    CheckAllowed(llvm::omp::Clause::Y); \
+    CheckAllowedClause(llvm::omp::Clause::Y); \
   }
 
 #define CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(X, Y) \
   void OmpStructureChecker::Enter(const parser::OmpClause::X &c) { \
-    CheckAllowed(llvm::omp::Clause::Y); \
+    CheckAllowedClause(llvm::omp::Clause::Y); \
     RequiresConstantPositiveParameter(llvm::omp::Clause::Y, c.v); \
   }
 
 #define CHECK_REQ_SCALAR_INT_CLAUSE(X, Y) \
   void OmpStructureChecker::Enter(const parser::OmpClause::X &c) { \
-    CheckAllowed(llvm::omp::Clause::Y); \
+    CheckAllowedClause(llvm::omp::Clause::Y); \
     RequiresPositiveParameter(llvm::omp::Clause::Y, c.v); \
   }
 
 // Use when clause don't falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_PARSER_CLAUSE(X, Y) \
   void OmpStructureChecker::Enter(const parser::X &) { \
-    CheckAllowed(llvm::omp::Y); \
+    CheckAllowedClause(llvm::omp::Y); \
   }
 
 // 'OmpWorkshareBlockChecker' is used to check the validity of the assignment
@@ -163,6 +163,43 @@ class AssociatedLoopChecker {
   std::map<std::string, std::int64_t> constructNamesAndLevels_;
 };
 
+bool OmpStructureChecker::CheckAllowedClause(llvmOmpClause clause) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  DirectiveContext &dirCtx = GetContext();
+  llvm::omp::Directive dir{dirCtx.directive};
+
+  if (!llvm::omp::isAllowedClauseForDirective(dir, clause, version)) {
+    unsigned allowedInVersion{[&] {
+      for (unsigned v : {45, 50, 51, 52, 60}) {
+        if (v <= version) {
+          continue;
+        }
+        if (llvm::omp::isAllowedClauseForDirective(dir, clause, v)) {
+          return v;
+        }
+      }
+      return 0u;
+    }()};
+
+    // Only report it if there is a later version that allows it.
+    // If it's not allowed at all, it will be reported by CheckAllowed.
+    if (allowedInVersion != 0) {
+      auto clauseName{parser::ToUpperCaseLetters(getClauseName(clause).str())};
+      auto dirName{parser::ToUpperCaseLetters(getDirectiveName(dir).str())};
+
+      std::string thisVersion{
+          std::to_string(version / 10) + "." + std::to_string(version % 10)};
+      std::string goodVersion{std::to_string(allowedInVersion)};
+
+      context_.Say(dirCtx.clauseSource,
+          "%s clause is not allowed on directive %s in OpenMP v%s, "
+          "try -fopenmp-version=%d"_err_en_US,
+          clauseName, dirName, thisVersion, allowedInVersion);
+    }
+  }
+  return CheckAllowed(clause);
+}
+
 bool OmpStructureChecker::IsCloselyNestedRegion(const OmpDirectiveSet &set) {
   // Definition of close nesting:
   //
@@ -1156,7 +1193,7 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclarativeAllocate &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Allocator &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_allocator);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_allocator);
   // Note: Predefined allocators are stored in ScalarExpr as numbers
   //   whereas custom allocators are stored as strings, so if the ScalarExpr
   //   actually has an int value, then it must be a predefined allocator
@@ -1165,7 +1202,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Allocator &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Allocate &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_allocate);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_allocate);
   if (const auto &modifier{
           std::get<std::optional<parser::OmpAllocateClause::AllocateModifier>>(
               x.v.t)}) {
@@ -1210,7 +1247,7 @@ void OmpStructureChecker::Leave(const parser::OmpDeclareTargetWithClause &x) {
         FindClause(llvm::omp::Clause::OMPC_link);
     if (!enterClause && !toClause && !linkClause) {
       context_.Say(x.source,
-          "If the DECLARE TARGET directive has a clause, it must contain at lease one ENTER clause or LINK clause"_err_en_US);
+          "If the DECLARE TARGET directive has a clause, it must contain at least one ENTER clause or LINK clause"_err_en_US);
     }
     if (toClause && context_.ShouldWarn(common::UsageWarning::OpenMPUsage)) {
       context_.Say(toClause->source,
@@ -2362,7 +2399,7 @@ CHECK_REQ_CONSTANT_SCALAR_INT_CLAUSE(Simdlen, OMPC_simdlen)
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_reduction);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_reduction);
   if (CheckReductionOperators(x)) {
     CheckReductionTypeList(x);
   }
@@ -2686,7 +2723,7 @@ void OmpStructureChecker::CheckSharedBindingInOuterContext(
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Ordered &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_ordered);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_ordered);
   // the parameter of ordered clause is optional
   if (const auto &expr{x.v}) {
     RequiresConstantPositiveParameter(llvm::omp::Clause::OMPC_ordered, *expr);
@@ -2701,17 +2738,17 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Ordered &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Shared &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_shared);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_shared);
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "SHARED");
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_private);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_private);
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "PRIVATE");
   CheckIntentInPointer(x.v, llvm::omp::Clause::OMPC_private);
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Nowait &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_nowait);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_nowait);
   if (llvm::omp::noWaitClauseNotAllowedSet.test(GetContext().directive)) {
     context_.Say(GetContext().clauseSource,
         "%s clause is not allowed on the OMP %s directive,"
@@ -2784,7 +2821,7 @@ void OmpStructureChecker::CheckIsVarPartOfAnotherVar(
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Firstprivate &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_firstprivate);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_firstprivate);
 
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "FIRSTPRIVATE");
   CheckIsLoopIvPartOfClause(llvmOmpClause::OMPC_firstprivate, x.v);
@@ -2871,7 +2908,7 @@ void OmpStructureChecker::Leave(const parser::OmpAtomic &) {
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 void OmpStructureChecker::Enter(const parser::OmpClause::Aligned &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_aligned);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_aligned);
 
   if (const auto &expr{
           std::get<std::optional<parser::ScalarIntConstantExpr>>(x.v.t)}) {
@@ -2880,7 +2917,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Aligned &x) {
   // 2.8.1 TODO: list-item attribute check
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Defaultmap &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_defaultmap);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_defaultmap);
   using VariableCategory = parser::OmpDefaultmapClause::VariableCategory;
   if (!std::get<std::optional<VariableCategory>>(x.v.t)) {
     context_.Say(GetContext().clauseSource,
@@ -2889,7 +2926,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Defaultmap &x) {
   }
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_if);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_if);
   using dirNameModifier = parser::OmpIfClause::DirectiveNameModifier;
   // TODO Check that, when multiple 'if' clauses are applied to a combined
   // construct, at most one of them applies to each directive.
@@ -2925,7 +2962,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::If &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Linear &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_linear);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_linear);
 
   // 2.7 Loop Construct Restriction
   if ((llvm::omp::allDoSet | llvm::omp::allSimdSet)
@@ -2959,7 +2996,7 @@ void OmpStructureChecker::CheckAllowedMapTypes(
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Map &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_map);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_map);
 
   if (const auto &maptype{std::get<std::optional<parser::OmpMapType>>(x.v.t)}) {
     using Type = parser::OmpMapType::Type;
@@ -3005,7 +3042,7 @@ bool OmpStructureChecker::ScheduleModifierHasType(
   return false;
 }
 void OmpStructureChecker::Enter(const parser::OmpClause::Schedule &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_schedule);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_schedule);
   const parser::OmpScheduleClause &scheduleClause = x.v;
 
   // 2.7 Loop Construct Restriction
@@ -3041,7 +3078,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Schedule &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Device &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_device);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_device);
   const parser::OmpDeviceClause &deviceClause = x.v;
   const auto &device{std::get<1>(deviceClause.t)};
   RequiresPositiveParameter(
@@ -3060,7 +3097,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Device &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Depend &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_depend);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_depend);
   if ((std::holds_alternative<parser::OmpDependClause::Source>(x.v.u) ||
           std::holds_alternative<parser::OmpDependClause::Sink>(x.v.u)) &&
       GetContext().directive != llvm::omp::OMPD_ordered) {
@@ -3103,7 +3140,7 @@ void OmpStructureChecker::CheckCopyingPolymorphicAllocatable(
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_copyprivate);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_copyprivate);
   CheckIntentInPointer(x.v, llvm::omp::Clause::OMPC_copyprivate);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
@@ -3121,7 +3158,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Copyprivate &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_lastprivate);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_lastprivate);
 
   CheckIsVarPartOfAnotherVar(GetContext().clauseSource, x.v, "LASTPRIVATE");
 
@@ -3145,7 +3182,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Lastprivate &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::Copyin &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_copyin);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_copyin);
 
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
@@ -3180,7 +3217,7 @@ void OmpStructureChecker::CheckStructureElement(
 
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
   CheckStructureElement(x.v, llvm::omp::Clause::OMPC_use_device_ptr);
-  CheckAllowed(llvm::omp::Clause::OMPC_use_device_ptr);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_use_device_ptr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
   semantics::UnorderedSymbolSet listVars;
@@ -3213,7 +3250,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::UseDevicePtr &x) {
 
 void OmpStructureChecker::Enter(const parser::OmpClause::UseDeviceAddr &x) {
   CheckStructureElement(x.v, llvm::omp::Clause::OMPC_use_device_addr);
-  CheckAllowed(llvm::omp::Clause::OMPC_use_device_addr);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_use_device_addr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
   semantics::UnorderedSymbolSet listVars;
@@ -3238,7 +3275,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::UseDeviceAddr &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_is_device_ptr);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_is_device_ptr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
   semantics::UnorderedSymbolSet listVars;
@@ -3276,7 +3313,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &x) {
 }
 
 void OmpStructureChecker::Enter(const parser::OmpClause::HasDeviceAddr &x) {
-  CheckAllowed(llvm::omp::Clause::OMPC_has_device_addr);
+  CheckAllowedClause(llvm::omp::Clause::OMPC_has_device_addr);
   SymbolSourceMap currSymbols;
   GetSymbolsInObjectList(x.v, currSymbols);
   semantics::UnorderedSymbolSet listVars;
@@ -3621,7 +3658,7 @@ void OmpStructureChecker::Enter(
 }
 
 void OmpStructureChecker::CheckAllowedRequiresClause(llvmOmpClause clause) {
-  CheckAllowed(clause);
+  CheckAllowedClause(clause);
 
   if (clause != llvm::omp::Clause::OMPC_atomic_default_mem_order) {
     // Check that it does not appear after a device construct
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 8bfd4d594b028..605f3f05b4bc8 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -139,6 +139,7 @@ class OmpStructureChecker
   }
 
 private:
+  bool CheckAllowedClause(llvmOmpClause clause);
   void CheckMultipleOccurrence(semantics::UnorderedSymbolSet &listVars,
       const std::list<parser::Name> &nameList, const parser::CharBlock &item,
       const std::string &clauseName);
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index 8592d1e5d6217..1f2980b07b3e0 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -348,9 +348,10 @@ class CommonBlockMap {
 SemanticsContext::SemanticsContext(
     const common::IntrinsicTypeDefaultKinds &defaultKinds,
     const common::LanguageFeatureControl &languageFeatures,
+    const common::LangOptions &langOpts,
     parser::AllCookedSources &allCookedSources)
     : defaultKinds_{defaultKinds}, languageFeatures_{languageFeatures},
-      allCookedSources_{allCookedSources},
+      langOpts_{langOpts}, allCookedSources_{allCookedSources},
       intrinsics_{evaluate::IntrinsicProcTable::Configure(defaultKinds_)},
       globalScope_{*this}, intrinsicModulesScope_{globalScope_.MakeScope(
                                Scope::Kind::IntrinsicModules, nullptr)},
diff --git a/flang/test/Driver/color-diagnostics-forwarding.f90 b/flang/test/Driver/color-diagnostics-forwarding.f90
index daef17cb75787..368fa8834142a 100644
--- a/flang/test/Driver/color-diagnostics-forwarding.f90
+++ b/flang/test/Driver/color-diagnostics-forwarding.f90
@@ -1,21 +1,53 @@
-! Test that flang-new forwards -f{no-}color-diagnostics options to
-! flang-new -fc1 as expected.
+! Test that flang-new forwards -f{no-}color-diagnostics and
+! -f{no-}diagnostics-color options to flang-new -fc1 as expected.
 
 ! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 -fcolor-diagnostics \
 ! RUN:   | FileCheck %s --check-prefix=CHECK-CD
+! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 -fdiagnostics-color \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD
+! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 -fdiagnostics-color=always \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD
 ! CHECK-CD: "-fc1"{{.*}} "-fcolor-diagnostics"
 
 ! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 -fno-color-diagnostics \
 ! RUN:   | FileCheck %s --check-prefix=CHECK-NCD
+! RUN: %flang -fsyntax-only -### %s -o %t -fno-diagnostics-color 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD
+! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 -fdiagnostics-color=never \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD
 ! CHECK-NCD-NOT: "-fc1"{{.*}} "-fcolor-diagnostics"
 
 ! Check that the last flag wins.
 ! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 \
 ! RUN:     -fno-color-diagnostics -fcolor-diagnostics \
 ! RUN:   | FileCheck %s --check-prefix=CHECK-NCD_CD_S
+! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 \
+! RUN:     -fno-diagnostics-color -fdiagnostics-color \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD_CD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fno-color-diagnostics -fdiagnostics-color=always 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD_CD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fdiagnostics-color=never -fdiagnostics-color=always 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD_CD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fdiagnostics-color=never -fcolor-diagnostics 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-NCD_CD_S
 ! CHECK-NCD_CD_S: "-fc1"{{.*}} "-fcolor-diagnostics"
 
 ! RUN: %flang -fsyntax-only -### %s -o %t 2>&1 \
 ! RUN:     -fcolor-diagnostics -fno-color-diagnostics \
 ! RUN:   | FileCheck %s --check-prefix=CHECK-CD_NCD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fdiagnostics-color -fno-diagnostics-color  2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD_NCD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fdiagnostics-color=always -fno-color-diagnostics 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD_NCD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fdiagnostics-color=always -fdiagnostics-color=never 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD_NCD_S
+! RUN: %flang -fsyntax-only -### %s -o %t \
+! RUN:     -fcolor-diagnostics -fdiagnostics-color=never 2>&1 \
+! RUN:   | FileCheck %s --check-prefix=CHECK-CD_NCD_S
 ! CHECK-CD_NCD_S-NOT: "-fc1"{{.*}} "-fcolor-diagnostics"
diff --git a/flang/test/Driver/color-diagnostics-parse.f90 b/flang/test/Driver/color-diagnostics-parse.f90
index 11a1c7b57c9e2..3682224ac9525 100644
--- a/flang/test/Driver/color-diagnostics-parse.f90
+++ b/flang/test/Driver/color-diagnostics-parse.f90
@@ -1,12 +1,22 @@
-! Test the behaviors of -f{no-}color-diagnostics when emitting parsing
-! diagnostics.
+! Test the behaviors of -f{no-}color-diagnostics and -f{no-}diagnostics-color
+! when emitting parsing diagnostics.
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
 ! RUN: not %flang %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fdiagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fdiagnostics-color=always 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+
 ! RUN: not %flang %s -fno-color-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang %s -fno-diagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang %s -fdiagnostics-color=never 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
 ! RUN: not %flang_fc1 %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 ! RUN: not %flang_fc1 %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
diff --git a/flang/test/Driver/color-diagnostics-scan.f b/flang/test/Driver/color-diagnostics-scan.f
index d901d77adaf8f..29d4635b4fb03 100644
--- a/flang/test/Driver/color-diagnostics-scan.f
+++ b/flang/test/Driver/color-diagnostics-scan.f
@@ -1,5 +1,5 @@
-! Test the behaviors of -f{no-}color-diagnostics when emitting scanning
-! diagnostics.
+! Test the behaviors of -f{no-}color-diagnostics and -f{no}-diagnostic-colors
+! when emitting scanning diagnostics.
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
@@ -9,6 +9,17 @@
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 ! RUN: not %flang_fc1 -E -Werror %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+
+! RUN: not %flang %s -E -Werror -fdiagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -E -Werror -fno-diagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=always 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -E -Werror -fdiagnostics-color=never 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
 ! RUN: not %flang_fc1 -E -Werror %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;35mwarning: {{.*}}[0mCharacter in fixed-form label field must be a digit
diff --git a/flang/test/Driver/color-diagnostics-sema.f90 b/flang/test/Driver/color-diagnostics-sema.f90
index df7a69f297f12..ca87b196a82f0 100644
--- a/flang/test/Driver/color-diagnostics-sema.f90
+++ b/flang/test/Driver/color-diagnostics-sema.f90
@@ -1,5 +1,5 @@
-! Test the behaviors of -f{no-}color-diagnostics when emitting semantic
-! diagnostics.
+! Test the behaviors of -f{no-}color-diagnostics and -f{no}diagnostics-color
+! when emitting semantic diagnostics.
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
@@ -9,6 +9,17 @@
 ! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
 ! RUN: not %flang_fc1 %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+
+! RUN: not %flang %s -fdiagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fno-diagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
+! RUN: not %flang %s -fdiagnostics-color=always 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fdiagnostics-color=never 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
 ! RUN: not %flang_fc1 %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;31merror: {{.*}}[0mMust be a constant value
diff --git a/flang/test/Driver/color-diagnostics.f90 b/flang/test/Driver/color-diagnostics.f90
index 2d18196d0af73..cbb6bf74f97f7 100644
--- a/flang/test/Driver/color-diagnostics.f90
+++ b/flang/test/Driver/color-diagnostics.f90
@@ -1,4 +1,4 @@
-! Test the behaviors of -f{no-}color-diagnostics.
+! Test the behaviors of -f{no-}color-diagnostics and -f{no}-diagnostics-color.
 ! Windows command prompt doesn't support ANSI escape sequences.
 ! REQUIRES: shell
 
@@ -9,14 +9,36 @@
 ! RUN: not %flang_fc1 %s -fcolor-diagnostics 2>&1 \
 ! RUN:     | FileCheck %s --check-prefix=CHECK_CD
 ! RUN: not %flang_fc1 %s -fno-color-diagnostics 2>&1 \
-! RUN:     | FileCheck %s --check-prefix=UNSUPPORTED_OPTION
+! RUN:     | FileCheck %s --check-prefix=UNSUPPORTED_COLOR_DIAGS
+
+! RUN: not %flang %s -fdiagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fno-diagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+! RUN: not %flang_fc1 %s -fdiagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=UNSUPPORTED_DIAGS_COLOR
+! RUN: not %flang_fc1 %s -fno-diagnostics-color 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=UNSUPPORTED_NO_DIAGS_COLOR
+
+! RUN: not %flang %s -fdiagnostics-color=always 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_CD
+! RUN: not %flang %s -fdiagnostics-color=never 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=CHECK_NCD
+
 ! RUN: not %flang_fc1 %s 2>&1 | FileCheck %s --check-prefix=CHECK_NCD
 
 ! CHECK_CD: {{.*}}[0;1;31merror: {{.*}}[0m{{.*}}[1mSemantic errors in {{.*}}color-diagnostics.f90{{.*}}[0m
 
 ! CHECK_NCD: Semantic errors in {{.*}}color-diagnostics.f90
 
-! UNSUPPORTED_OPTION: error: unknown argument: '-fno-color-diagnostics'
+! UNSUPPORTED_COLOR_DIAGS: error: unknown argument: '-fno-color-diagnostics'
+! UNSUPPORTED_DIAGS_COLOR: error: unknown argument: '-fdiagnostics-color'
+! UNSUPPORTED_NO_DIAGS_COLOR: error: unknown argument: '-fno-diagnostics-color'
+
+! Check that invalid values of -fdiagnostics-color= are disallowed.
+! RUN: not %flang %s -fdiagnostics-color=sometimes 2>&1 \
+! RUN:     | FileCheck %s --check-prefix=DCEQ_BAD
+! DCEQ_BAD: error: invalid argument 'sometimes' to -fdiagnostics-color=
 
 program m
   integer :: i = k
diff --git a/flang/test/Driver/use-module-error.f90 b/flang/test/Driver/use-module-error.f90
index 42d6650621c8c..0b47b682d938c 100644
--- a/flang/test/Driver/use-module-error.f90
+++ b/flang/test/Driver/use-module-error.f90
@@ -3,6 +3,11 @@
 !--------------------------
 ! FLANG DRIVER (flang-new)
 !--------------------------
+! RUN: %flang -fsyntax-only -J %S/Inputs/ %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang -fsyntax-only -J %S/Inputs/ -J %S/Inputs/ %s  2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang -fsyntax-only -module-dir %S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang -fsyntax-only -module-dir %S/Inputs/module-dir -module-dir %S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang -fsyntax-only -module-dir %S/Inputs/module-dir -J%S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
 ! RUN: not %flang -fsyntax-only -J %S/Inputs/module-dir -J %S/Inputs/ %s  2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
 ! RUN: not %flang -fsyntax-only -J %S/Inputs/module-dir -module-dir %S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
 ! RUN: not %flang -fsyntax-only -module-dir %S/Inputs/module-dir -J%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
@@ -10,11 +15,17 @@
 !-----------------------------------------
 ! FRONTEND FLANG DRIVER (flang-new -fc1)
 !-----------------------------------------
+! RUN: %flang_fc1 -fsyntax-only -J %S/Inputs/ %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang_fc1 -fsyntax-only -J %S/Inputs/ -J %S/Inputs/ %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang_fc1 -fsyntax-only -module-dir %S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty  --check-prefix=SINGLEINCLUDE
+! RUN: %flang_fc1 -fsyntax-only -module-dir %S/Inputs/module-dir -module-dir %S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
+! RUN: %flang_fc1 -fsyntax-only -module-dir %S/Inputs/module-dir -J%S/Inputs/module-dir %s 2>&1 | FileCheck %s --allow-empty --check-prefix=SINGLEINCLUDE
 ! RUN: not %flang_fc1 -fsyntax-only -J %S/Inputs/module-dir -J %S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
 ! RUN: not %flang_fc1 -fsyntax-only -J %S/Inputs/module-dir -module-dir %S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
 ! RUN: not %flang_fc1 -fsyntax-only -module-dir %S/Inputs/module-dir -J%S/Inputs/ %s 2>&1 | FileCheck %s --check-prefix=DOUBLEINCLUDE
 
-! DOUBLEINCLUDE:error: Only one '-module-dir/-J' option allowed
+! DOUBLEINCLUDE:error: Only one '-module-dir/-J' directory allowed
+! SINGLEINCLUDE-NOT:error: Only one '-module-dir/-J' directory allowed
 
 program too_many_module_dirs
 end
diff --git a/flang/test/Driver/xoffload-linker.f90 b/flang/test/Driver/xoffload-linker.f90
new file mode 100644
index 0000000000000..c8988de1fdec8
--- /dev/null
+++ b/flang/test/Driver/xoffload-linker.f90
@@ -0,0 +1,10 @@
+! Test the -Xoffload-linker flag that forwards link commands to the clang-linker-wrapper used
+! to help link offloading device libraries
+
+! RUN: %flang -### %s -o %t 2>&1 -fopenmp --offload-arch=gfx90a --target=aarch64-unknown-linux-gnu -nogpulib -Xoffload-linker a | FileCheck %s --check-prefix=CHECK-XLINKER
+
+! CHECK-XLINKER: "{{[^"]*}}clang-linker-wrapper{{.*}}"{{.*}}"--device-linker=a"{{.*}}
+
+! RUN: %flang -### %s -o %t 2>&1 -fopenmp --offload-arch=gfx90a --target=aarch64-unknown-linux-gnu -nogpulib -Xoffload-linker a -Xoffload-linker-amdgcn-amd-amdhsa b | FileCheck %s --check-prefix=CHECK-XLINKER-AMDGCN
+
+! CHECK-XLINKER-AMDGCN: "{{[^"]*}}clang-linker-wrapper{{.*}}"{{.*}}"--device-linker=a"{{.*}}"--device-linker=amdgcn-amd-amdhsa=b"{{.*}}
diff --git a/flang/test/Examples/omp-in-reduction-clause.f90 b/flang/test/Examples/omp-in-reduction-clause.f90
index cf06b6dc1a8a0..fc3fff549a8d2 100644
--- a/flang/test/Examples/omp-in-reduction-clause.f90
+++ b/flang/test/Examples/omp-in-reduction-clause.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: plugins, examples, shell
 
-! RUN: %flang_fc1 -load %llvmshlibdir/flangOmpReport.so -plugin flang-omp-report -fopenmp  %s -o - | FileCheck %s
+! RUN: %flang_fc1 -load %llvmshlibdir/flangOmpReport.so -plugin flang-omp-report -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
 
 ! Check for IN_REDUCTION() clause on OpenMP constructs
 
diff --git a/flang/test/Examples/omp-order-clause.f90 b/flang/test/Examples/omp-order-clause.f90
index 97c0707b18810..8d1c3f4e59137 100644
--- a/flang/test/Examples/omp-order-clause.f90
+++ b/flang/test/Examples/omp-order-clause.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: plugins, examples, shell
 
-! RUN: %flang_fc1 -load %llvmshlibdir/flangOmpReport.so -plugin flang-omp-report -fopenmp  %s -o - | FileCheck %s
+! RUN: %flang_fc1 -load %llvmshlibdir/flangOmpReport.so -plugin flang-omp-report -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
 
 ! Check for ORDER([order-modifier :]concurrent) clause on OpenMP constructs
 
diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf
index bed0a4574fe94..1331b644130c8 100644
--- a/flang/test/Lower/CUDA/cuda-device-proc.cuf
+++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf
@@ -26,11 +26,11 @@ end
 ! CHECK: %{{.*}} = fir.call @__syncthreads_count(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (!fir.ref<i32>) -> i32
 ! CHECK: %{{.*}} = fir.call @__syncthreads_or(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (!fir.ref<i32>) -> i32
 
-! CHECK: func.func private @__syncthreads() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads"}
-! CHECK: func.func private @__syncwarp(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncwarp"}
-! CHECK: func.func private @__threadfence() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence"}
-! CHECK: func.func private @__threadfence_block() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence_block"}
-! CHECK: func.func private @__threadfence_system() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence_system"}
-! CHECK: func.func private @__syncthreads_and(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_and"}
-! CHECK: func.func private @__syncthreads_count(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_count"}
-! CHECK: func.func private @__syncthreads_or(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_or"}
+! CHECK: func.func private @__syncthreads() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__syncwarp(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncwarp", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__threadfence() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__threadfence_block() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence_block", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__threadfence_system() attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__threadfence_system", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__syncthreads_and(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_and", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__syncthreads_count(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_count", fir.proc_attrs = #fir.proc_attrs<bind_c>}
+! CHECK: func.func private @__syncthreads_or(!fir.ref<i32> {cuf.data_attr = #cuf.cuda<device>}) -> i32 attributes {cuf.proc_attr = #cuf.cuda_proc<device>, fir.bindc_name = "__syncthreads_or", fir.proc_attrs = #fir.proc_attrs<bind_c>}
diff --git a/flang/test/Lower/HLFIR/bindc-value-derived.f90 b/flang/test/Lower/HLFIR/bindc-value-derived.f90
index a54b29b470e0b..7a2196dfc8bf1 100644
--- a/flang/test/Lower/HLFIR/bindc-value-derived.f90
+++ b/flang/test/Lower/HLFIR/bindc-value-derived.f90
@@ -14,7 +14,7 @@ subroutine test(x) bind(c)
     call use_it(x%i)
   end subroutine
 ! CHECK-LABEL:   func.func @test(
-! CHECK-SAME:                    %[[VAL_0:.*]]: !fir.type<_QMbindc_byvalTt{i:i32}> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test"} {
+! CHECK-SAME:                    %[[VAL_0:.*]]: !fir.type<_QMbindc_byvalTt{i:i32}>
 ! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QMbindc_byvalTt{i:i32}>
 ! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>
 ! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %{{[0-9]+}} {fortran_attrs = #fir.var_attrs<value>, uniq_name = "_QMbindc_byvalFtestEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
@@ -28,7 +28,7 @@ subroutine call_it(x)
     call test(x)
   end subroutine
 ! CHECK-LABEL:   func.func @_QMbindc_byvalPcall_it(
-! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>> {fir.bindc_name = "x"}) {
+! CHECK-SAME:                                      %[[VAL_0:.*]]: !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>
 ! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %{{[0-9]+}} {uniq_name = "_QMbindc_byvalFcall_itEx"} : (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>, !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>)
 ! CHECK:           %[[VAL_2:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.type<_QMbindc_byvalTt{i:i32}>>
 ! CHECK:           fir.call @test(%[[VAL_2]]) proc_attrs<bind_c> fastmath<contract> : (!fir.type<_QMbindc_byvalTt{i:i32}>) -> ()
diff --git a/flang/test/Lower/HLFIR/block_bindc_pocs.f90 b/flang/test/Lower/HLFIR/block_bindc_pocs.f90
index ed07d88c53a60..fc04226dfd23d 100644
--- a/flang/test/Lower/HLFIR/block_bindc_pocs.f90
+++ b/flang/test/Lower/HLFIR/block_bindc_pocs.f90
@@ -11,7 +11,7 @@ end module m
 !CHECK-DAG: %[[S0:.*]] = llvm.intr.stacksave : !llvm.ptr
 !CHECK-DAG: fir.call @test_proc() proc_attrs<bind_c> fastmath<contract> : () -> ()
 !CHECK-DAG: llvm.intr.stackrestore %[[S0]] : !llvm.ptr
-!CHECK-DAG: func.func private @test_proc() attributes {fir.bindc_name = "test_proc"}
+!CHECK-DAG: func.func private @test_proc() attributes {fir.bindc_name = "test_proc", fir.proc_attrs = #fir.proc_attrs<bind_c>}
 subroutine test
     BLOCK
         use m
diff --git a/flang/test/Lower/Intrinsics/signal.f90 b/flang/test/Lower/Intrinsics/signal.f90
index 5d20bb5c5c074..39fef122d7754 100644
--- a/flang/test/Lower/Intrinsics/signal.f90
+++ b/flang/test/Lower/Intrinsics/signal.f90
@@ -4,14 +4,14 @@
 module m
 contains
 ! CHECK-LABEL:   func.func @handler(
-! CHECK-SAME:                       %[[VAL_0:.*]]: i32 {fir.bindc_name = "signum"}) attributes {fir.bindc_name = "handler"} {
+! CHECK-SAME:                       %[[VAL_0:.*]]: i32
   subroutine handler(signum) bind(C)
     use iso_c_binding, only: c_int
     integer(c_int), value :: signum
   end subroutine
 
 ! CHECK-LABEL:   func.func @_QMmPsetup_signals(
-! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i32> {fir.bindc_name = "optional_status", fir.optional}) {
+! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i32>
   subroutine setup_signals(optional_status)
     ! not portable accross systems
     integer, parameter :: SIGFPE = 8
diff --git a/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90 b/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90
new file mode 100644
index 0000000000000..eb60e7ff9858c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/DelayedPrivatization/distribute-standalone-private.f90
@@ -0,0 +1,42 @@
+! RUN: %flang_fc1 -emit-hlfir -fopenmp -mmlir --openmp-enable-delayed-privatization-staging \
+! RUN:   -o - %s 2>&1 | FileCheck %s
+! RUN: bbc -emit-hlfir -fopenmp --openmp-enable-delayed-privatization-staging -o - %s 2>&1 \
+! RUN:   | FileCheck %s
+
+subroutine standalone_distribute
+    implicit none
+    integer :: simple_var, i
+
+    !$omp teams
+    !$omp distribute private(simple_var)
+    do i = 1, 10
+      simple_var = simple_var + i
+    end do
+    !$omp end distribute
+    !$omp end teams
+end subroutine standalone_distribute
+
+! CHECK: omp.private {type = private} @[[I_PRIVATIZER_SYM:.*]] : !fir.ref<i32>
+! CHECK: omp.private {type = private} @[[VAR_PRIVATIZER_SYM:.*]] : !fir.ref<i32>
+
+
+! CHECK-LABEL: func.func @_QPstandalone_distribute() {
+! CHECK:         %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFstandalone_distributeEi"}
+! CHECK:         %[[VAR_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "_QFstandalone_distributeEsimple_var"}
+! CHECK:         omp.teams {
+! CHECK:           omp.distribute
+! CHECK-SAME:        private(@[[VAR_PRIVATIZER_SYM]] %[[VAR_DECL]]#0 -> %[[VAR_ARG:.*]] : !fir.ref<i32>,
+! CHECK-SAME:                @[[I_PRIVATIZER_SYM]] %[[I_DECL]]#0 -> %[[I_ARG:.*]] : !fir.ref<i32>) {
+! CHECK:             omp.loop_nest {{.*}} {
+! CHECK:               %[[VAR_PRIV_DECL:.*]]:2 = hlfir.declare %[[VAR_ARG]]
+! CHECK:               %[[I_PRIV_DECL:.*]]:2 = hlfir.declare %[[I_ARG]]
+
+! CHECK:               fir.store %{{.*}} to %[[I_PRIV_DECL]]#1 : !fir.ref<i32>
+! CHECK:               %{{.*}} = fir.load %[[VAR_PRIV_DECL]]#0 : !fir.ref<i32>
+! CHECK:               %{{.*}} = fir.load %[[I_PRIV_DECL]]#0 : !fir.ref<i32>
+! CHECK:               arith.addi %{{.*}}, %{{.*}} : i32
+! CHECK:               hlfir.assign %{{.*}} to %[[VAR_PRIV_DECL]]#0 : i32, !fir.ref<i32>
+! CHECK:             }
+! CHECK:           }
+! CHECK:         }
+! CHECK:       }
diff --git a/flang/test/Lower/OpenMP/atomic-capture.f90 b/flang/test/Lower/OpenMP/atomic-capture.f90
index 667ae8ed7a133..af82e4b2a20eb 100644
--- a/flang/test/Lower/OpenMP/atomic-capture.f90
+++ b/flang/test/Lower/OpenMP/atomic-capture.f90
@@ -2,8 +2,8 @@
 
 ! This test checks the lowering of atomic capture
 
-! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -fopenmp-version=50 -emit-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags -fopenmp-version=50 %s -o - | FileCheck %s
 
 
 program OmpAtomicCapture
diff --git a/flang/test/Lower/OpenMP/atomic-read.f90 b/flang/test/Lower/OpenMP/atomic-read.f90
index d578df959a474..c3270dd6c1d67 100644
--- a/flang/test/Lower/OpenMP/atomic-read.f90
+++ b/flang/test/Lower/OpenMP/atomic-read.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -fopenmp-version=50 -emit-hlfir %s -o - | FileCheck %s
 
 ! This test checks the lowering of atomic read
 
diff --git a/flang/test/Lower/OpenMP/atomic-update.f90 b/flang/test/Lower/OpenMP/atomic-update.f90
index 85edfdf4de84d..16dae9d5f301c 100644
--- a/flang/test/Lower/OpenMP/atomic-update.f90
+++ b/flang/test/Lower/OpenMP/atomic-update.f90
@@ -1,8 +1,8 @@
 ! REQUIRES: openmp_runtime
 
 ! This test checks lowering of atomic and atomic update constructs
-! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
-! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -fopenmp-version=50 -emit-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags -fopenmp-version=50 %s -o - | FileCheck %s
 
 program OmpAtomicUpdate
     use omp_lib
diff --git a/flang/test/Lower/OpenMP/atomic-write.f90 b/flang/test/Lower/OpenMP/atomic-write.f90
index 8867dc5921192..b30dc483e6b84 100644
--- a/flang/test/Lower/OpenMP/atomic-write.f90
+++ b/flang/test/Lower/OpenMP/atomic-write.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: bbc %openmp_flags -emit-hlfir %s -o - | FileCheck %s
+! RUN: bbc %openmp_flags -fopenmp-version=50 -emit-hlfir %s -o - | FileCheck %s
 
 ! This test checks the lowering of atomic write
 
diff --git a/flang/test/Lower/OpenMP/declare-target-data.f90 b/flang/test/Lower/OpenMP/declare-target-data.f90
index d86f74d18b6df..154853a0fa20c 100644
--- a/flang/test/Lower/OpenMP/declare-target-data.f90
+++ b/flang/test/Lower/OpenMP/declare-target-data.f90
@@ -1,5 +1,5 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s 
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s 
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s
 
 module test_0
     implicit none
diff --git a/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
index 1998c3da23af5..079d43e309028 100644
--- a/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
+++ b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
@@ -1,5 +1,5 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s --check-prefixes ALL,HOST
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s --check-prefixes ALL
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s --check-prefixes ALL,HOST
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-device %s -o - | FileCheck %s --check-prefixes ALL
 
 program main
     use, intrinsic ::  iso_c_binding
diff --git a/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90 b/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
index 0d138321445ce..db8320a598052 100644
--- a/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
+++ b/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
@@ -1,5 +1,5 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s --check-prefixes ALL,HOST
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s --check-prefixes ALL,DEVICE
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s --check-prefixes ALL,HOST
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-device %s -o - | FileCheck %s --check-prefixes ALL,DEVICE
 
 ! Check specification valid forms of declare target with functions 
 ! utilising device_type and to clauses as well as the default 
@@ -154,7 +154,7 @@ SUBROUTINE SUBR_DEFAULT_EXTENDEDLIST()
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPrecursive_declare_target
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
 RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET(INCREMENT) RESULT(K)
 !$omp declare target to(RECURSIVE_DECLARE_TARGET) device_type(nohost)
     INTEGER :: INCREMENT, K
@@ -166,7 +166,7 @@ RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET(INCREMENT) RESULT(K)
 END FUNCTION RECURSIVE_DECLARE_TARGET
 
 ! DEVICE-LABEL: func.func @_QPrecursive_declare_target_enter
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
 RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET_ENTER(INCREMENT) RESULT(K)
 !$omp declare target enter(RECURSIVE_DECLARE_TARGET_ENTER) device_type(nohost)
     INTEGER :: INCREMENT, K
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
index 0ca2bcbd66a96..941f1eecbaf56 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
@@ -1,7 +1,7 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s  --check-prefix=DEVICE
-!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s  --check-prefix=DEVICE
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_twice
 ! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
@@ -105,7 +105,7 @@ end function target_function_test_host
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_with_dev_type_recursive
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
 recursive function implicitly_captured_with_dev_type_recursive(increment) result(k)
 !$omp declare target enter(implicitly_captured_with_dev_type_recursive) device_type(host)
    integer :: increment, k
@@ -174,7 +174,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end program
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
 recursive subroutine implicitly_captured_recursive(increment)
    integer :: increment
    if (increment == 10) then
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
index ffca5c3ff2500..8140fcc5f4813 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
@@ -1,7 +1,7 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s  --check-prefix=DEVICE
-!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=50 -fopenmp-is-target-device %s -o - | FileCheck %s  --check-prefix=DEVICE
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=50 %s -o - | FileCheck %s
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=50 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured
 ! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
@@ -131,7 +131,7 @@ end function target_function_test_host
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_with_dev_type_recursive
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
 recursive function implicitly_captured_with_dev_type_recursive(increment) result(k)
 !$omp declare target to(implicitly_captured_with_dev_type_recursive) device_type(host)
    integer :: increment, k
@@ -200,7 +200,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end program
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
 recursive subroutine implicitly_captured_recursive(increment)
    integer :: increment
    if (increment == 10) then
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
index 9b85a32036ca5..eca527ffffcec 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
@@ -1,7 +1,7 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-is-device %s -o - | FileCheck %s  --check-prefix=DEVICE
-!RUN: bbc -emit-hlfir -fopenmp %s -o - | FileCheck %s
-!RUN: bbc -emit-hlfir -fopenmp -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-device %s -o - | FileCheck %s  --check-prefix=DEVICE
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+!RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! DEVICE-LABEL: func.func @_QPimplicit_capture
 ! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
@@ -67,7 +67,7 @@ end function target_function_test_device
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {fir.func_recursive, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
 recursive function implicitly_captured_recursive(increment) result(k)
    integer :: increment, k
    if (increment == 10) then
diff --git a/flang/test/Lower/OpenMP/function-filtering-2.f90 b/flang/test/Lower/OpenMP/function-filtering-2.f90
index f367069efb3d9..0c02aa223820e 100644
--- a/flang/test/Lower/OpenMP/function-filtering-2.f90
+++ b/flang/test/Lower/OpenMP/function-filtering-2.f90
@@ -1,9 +1,9 @@
-! RUN: %flang_fc1 -fopenmp -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-HOST %s
-! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
-! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-DEVICE %s
-! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
-! RUN: bbc -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-HOST %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM,LLVM-DEVICE %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefix=MLIR %s
+! RUN: bbc -fopenmp -fopenmp-version=52 -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
+! RUN: bbc -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
 
 ! MLIR: func.func @{{.*}}implicit_invocation() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>}
 ! MLIR: return
diff --git a/flang/test/Lower/OpenMP/function-filtering.f90 b/flang/test/Lower/OpenMP/function-filtering.f90
index c473b9961907d..a72822503c373 100644
--- a/flang/test/Lower/OpenMP/function-filtering.f90
+++ b/flang/test/Lower/OpenMP/function-filtering.f90
@@ -1,9 +1,9 @@
-! RUN: %flang_fc1 -fopenmp -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM-HOST,LLVM-ALL %s
-! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
-! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM-DEVICE,LLVM-ALL %s
-! RUN: %flang_fc1 -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
-! RUN: bbc -fopenmp -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
-! RUN: bbc -fopenmp -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM-HOST,LLVM-ALL %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -flang-experimental-hlfir -emit-llvm %s -o - | FileCheck --check-prefixes=LLVM-DEVICE,LLVM-ALL %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
+! RUN: bbc -fopenmp -fopenmp-version=52 -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
+! RUN: bbc -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s
 
 ! Check that the correct LLVM IR functions are kept for the host and device
 ! after running the whole set of translation and transformation passes from
diff --git a/flang/test/Lower/bindc_procs.f90 b/flang/test/Lower/bindc_procs.f90
index 514f7713c383b..232e9d809bf17 100644
--- a/flang/test/Lower/bindc_procs.f90
+++ b/flang/test/Lower/bindc_procs.f90
@@ -1,6 +1,6 @@
 ! RUN: bbc -emit-fir %s -o - | FileCheck %s
 
-! CHECK-DAG: func.func private @proc1() attributes {fir.bindc_name = "proc1"}
+! CHECK-DAG: func.func private @proc1() attributes {fir.bindc_name = "proc1", fir.proc_attrs = #fir.proc_attrs<bind_c>}
 module decl1
   interface
      subroutine proc_iface() bind(C)
@@ -13,7 +13,7 @@ subroutine test1(x)
   call PrOc1
 end subroutine test1
 
-! CHECK-DAG: func.func private @proc2() attributes {fir.bindc_name = "proc2"}
+! CHECK-DAG: func.func private @proc2() attributes {fir.bindc_name = "proc2", fir.proc_attrs = #fir.proc_attrs<bind_c>}
 module decl2
   interface
      subroutine proc_iface() bind(C)
@@ -26,7 +26,7 @@ subroutine test2(x)
   call PrOc2
 end subroutine test2
 
-! CHECK-DAG: func.func private @func3() -> f32 attributes {fir.bindc_name = "func3"}
+! CHECK-DAG: func.func private @func3() -> f32 attributes {fir.bindc_name = "func3", fir.proc_attrs = #fir.proc_attrs<bind_c>}
 module decl3
   interface
      real function func_iface() bind(C)
@@ -40,7 +40,7 @@ subroutine test3(x)
   x = FuNc3()
 end subroutine test3
 
-! CHECK-DAG: func.func private @func4() -> f32 attributes {fir.bindc_name = "func4"}
+! CHECK-DAG: func.func private @func4() -> f32 attributes {fir.bindc_name = "func4", fir.proc_attrs = #fir.proc_attrs<bind_c>}
 module decl4
   interface
      real function func_iface() bind(C)
diff --git a/flang/test/Lower/c-interoperability-c-pointer.f90 b/flang/test/Lower/c-interoperability-c-pointer.f90
index 780e3d7dbcb68..9700440f6650b 100644
--- a/flang/test/Lower/c-interoperability-c-pointer.f90
+++ b/flang/test/Lower/c-interoperability-c-pointer.f90
@@ -32,7 +32,7 @@ subroutine c_func(c_t1, c_t2) bind(c, name="c_func")
 end
 
 ! CHECK-LABEL: func.func @test_callee_c_ptr(
-! CHECK-SAME:                               %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "ptr1"}) attributes {fir.bindc_name = "test_callee_c_ptr"} {
+! CHECK-SAME:                               %[[VAL_0:.*]]: !fir.ref<i64>
 ! CHECK:         %[[VAL_5:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> {bindc_name = "local", uniq_name = "_QFtest_callee_c_ptrElocal"}
 ! CHECK:         %[[VAL_1:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
 ! CHECK:         %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
@@ -56,7 +56,7 @@ subroutine test_callee_c_ptr(ptr1) bind(c)
 end subroutine
 
 ! CHECK-LABEL: func.func @test_callee_c_funptr(
-! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "ptr1"}) attributes {fir.bindc_name = "test_callee_c_funptr"} {
+! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i64>
 ! CHECK:         %[[VAL_5:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}> {bindc_name = "local", uniq_name = "_QFtest_callee_c_funptrElocal"}
 ! CHECK:         %[[VAL_1:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
 ! CHECK:         %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_funptr{__address:i64}>
diff --git a/flang/test/Lower/call.f90 b/flang/test/Lower/call.f90
index 78e9b5f4bc8a7..dc5120c6eb226 100644
--- a/flang/test/Lower/call.f90
+++ b/flang/test/Lower/call.f90
@@ -45,7 +45,7 @@ function f_int_to_char(i) bind(c, name="f_int_to_char")
 end function
 
 ! CHECK-LABEL: func.func @f_int_to_char(
-! CHECK-SAME: %[[ARG0:.*]]: i32 {fir.bindc_name = "i"}) -> !fir.char<1> attributes {fir.bindc_name = "f_int_to_char"} {
+! CHECK-SAME: %[[ARG0:.*]]: i32 {fir.bindc_name = "i"}) -> !fir.char<1> attributes {fir.bindc_name = "f_int_to_char", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
 ! CHECK: %[[CHARBOX:.*]] = fir.alloca !fir.char<1> {adapt.valuebyref}
 ! CHECK: %[[RESULT:.*]] = fir.alloca !fir.char<1> {bindc_name = "f_int_to_char", uniq_name = "_QFf_int_to_charEf_int_to_char"}
 ! CHECK: %[[INT_I:.*]] = fir.alloca i32
diff --git a/flang/test/Lower/func-attrs.f90 b/flang/test/Lower/func-attrs.f90
index 7ab549a0ac7ce..6c1e70bf6dabb 100644
--- a/flang/test/Lower/func-attrs.f90
+++ b/flang/test/Lower/func-attrs.f90
@@ -3,29 +3,34 @@
 pure subroutine sub1()
 end
 
-! CHECK: func.func @_QPsub1() attributes {fir.func_pure}
+! CHECK: func.func @_QPsub1() attributes {fir.proc_attrs = #fir.proc_attrs<pure>}
 
 elemental subroutine sub2()
 end
 
-! CHECK: func.func @_QPsub2() attributes {fir.func_elemental, fir.func_pure}
+! CHECK: func.func @_QPsub2() attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
 
-recursive subroutine sub3()
+non_recursive subroutine sub3()
 end
 
-! CHECK: func.func @_QPsub3() attributes {fir.func_recursive}
+! CHECK: func.func @_QPsub3() attributes {fir.proc_attrs = #fir.proc_attrs<non_recursive>}
+
+impure elemental subroutine sub4()
+end
+
+! CHECK: func.func @_QPsub4() attributes {fir.proc_attrs = #fir.proc_attrs<elemental>}
 
 pure function fct1()
 end
 
-! CHECK: func.func @_QPfct1() -> f32 attributes {fir.func_pure}
+! CHECK: func.func @_QPfct1() -> f32 attributes {fir.proc_attrs = #fir.proc_attrs<pure>}
 
 elemental function fct2()
 end
 
-! CHECK: func.func @_QPfct2() -> f32 attributes {fir.func_elemental, fir.func_pure}
+! CHECK: func.func @_QPfct2() -> f32 attributes {fir.proc_attrs = #fir.proc_attrs<elemental, pure>}
 
-recursive function fct3()
+non_recursive function fct3()
 end
 
-! CHECK: func.func @_QPfct3() -> f32 attributes {fir.func_recursive}
+! CHECK: func.func @_QPfct3() -> f32 attributes {fir.proc_attrs = #fir.proc_attrs<non_recursive>}
diff --git a/flang/test/Lower/host-associated.f90 b/flang/test/Lower/host-associated.f90
index 67465f5a7073d..9b4269df7bfcb 100644
--- a/flang/test/Lower/host-associated.f90
+++ b/flang/test/Lower/host-associated.f90
@@ -309,7 +309,7 @@ subroutine test7(j, k)
 contains
 
 ! CHECK-LABEL: func private @_QFtest7Ptest7_inner(
-! CHECK-SAME: %[[i:.*]]: !fir.ref<i32>{{.*}}, %[[tup:.*]]: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc}) -> i32 attributes {fir.func_elemental, fir.func_pure, fir.host_symbol = {{.*}}, llvm.linkage = #llvm.linkage<internal>} {
+! CHECK-SAME: %[[i:.*]]: !fir.ref<i32>{{.*}}, %[[tup:.*]]: !fir.ref<tuple<!fir.ref<i32>>> {fir.host_assoc}) -> i32 attributes {fir.host_symbol = {{.*}}, fir.proc_attrs = #fir.proc_attrs<elemental, pure>, llvm.linkage = #llvm.linkage<internal>} {
 elemental integer function test7_inner(i)
   implicit none
   integer, intent(in) :: i
diff --git a/flang/test/Lower/program-units-fir-mangling.f90 b/flang/test/Lower/program-units-fir-mangling.f90
index 002343c45f6ec..e0af6f065f34d 100644
--- a/flang/test/Lower/program-units-fir-mangling.f90
+++ b/flang/test/Lower/program-units-fir-mangling.f90
@@ -134,22 +134,22 @@ subroutine should_not_collide()
 end subroutine
 end program
 
-! CHECK-LABEL: func @omp_get_num_threads() -> f32 attributes {fir.bindc_name = "omp_get_num_threads"} {
+! CHECK-LABEL: func @omp_get_num_threads() -> f32 attributes {fir.bindc_name = "omp_get_num_threads", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
 function omp_get_num_threads() bind(c)
 ! CHECK: }
 end function
 
-! CHECK-LABEL: func @get_threads() -> f32 attributes {fir.bindc_name = "get_threads"} {
+! CHECK-LABEL: func @get_threads() -> f32 attributes {fir.bindc_name = "get_threads", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
 function omp_get_num_threads_1() bind(c, name ="get_threads")
 ! CHECK: }
 end function
 
-! CHECK-LABEL: func @bEtA() -> f32 attributes {fir.bindc_name = "bEtA"} {
+! CHECK-LABEL: func @bEtA() -> f32 attributes {fir.bindc_name = "bEtA", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
 function alpha() bind(c, name =" bEtA ")
 ! CHECK: }
 end function
 
-! CHECK-LABEL: func @bc1() attributes {fir.bindc_name = "bc1"} {
+! CHECK-LABEL: func @bc1() attributes {fir.bindc_name = "bc1", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
 subroutine bind_c_s() Bind(C,Name='bc1')
   ! CHECK: return
 end subroutine bind_c_s
@@ -175,11 +175,11 @@ subroutine bind_c_s() Bind(C, name='bc1')
 ! Test that BIND(C) label is taken into account for ENTRY symbols.
 ! CHECK-LABEL: func @_QPsub_with_entries() {
 subroutine sub_with_entries
-! CHECK-LABEL: func @bar() attributes {fir.bindc_name = "bar"} {
+! CHECK-LABEL: func @bar() attributes {fir.bindc_name = "bar", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
  entry some_entry() bind(c, name="bar")
 ! CHECK-LABEL: func @_QPnormal_entry() {
  entry normal_entry()
-! CHECK-LABEL: func @some_other_entry() attributes {fir.bindc_name = "some_other_entry"} {
+! CHECK-LABEL: func @some_other_entry() attributes {fir.bindc_name = "some_other_entry", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
  entry some_other_entry() bind(c)
 end subroutine
 
@@ -196,24 +196,24 @@ subroutine s1() bind(c,name=ok//'2')
     end subroutine
   end interface
  contains
-! CHECK-LABEL: func @ok3() -> f32 attributes {fir.bindc_name = "ok3"} {
+! CHECK-LABEL: func @ok3() -> f32 attributes {fir.bindc_name = "ok3", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
   real function f2() bind(c,name=foo//'3')
     character*(*), parameter :: foo = ok
 ! CHECK: fir.call @ok1() {{.*}}: () -> f32
-! CHECK-LABEL: func @ok4() -> f32 attributes {fir.bindc_name = "ok4"} {
+! CHECK-LABEL: func @ok4() -> f32 attributes {fir.bindc_name = "ok4", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
     entry f3() bind(c,name=foo//'4')
 ! CHECK: fir.call @ok1() {{.*}}: () -> f32
     f2 = f1()
   end function
-! CHECK-LABEL: func @ok5() attributes {fir.bindc_name = "ok5"} {
+! CHECK-LABEL: func @ok5() attributes {fir.bindc_name = "ok5", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
   subroutine s2() bind(c,name=foo//'5')
     character*(*), parameter :: foo = ok
 ! CHECK: fir.call @ok2() {{.*}}: () -> ()
-! CHECK-LABEL: func @ok6() attributes {fir.bindc_name = "ok6"} {
+! CHECK-LABEL: func @ok6() attributes {fir.bindc_name = "ok6", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
     entry s3() bind(c,name=foo//'6')
 ! CHECK: fir.call @ok2() {{.*}}: () -> ()
     continue ! force end of specification part
-! CHECK-LABEL: func @ok7() attributes {fir.bindc_name = "ok7"} {
+! CHECK-LABEL: func @ok7() attributes {fir.bindc_name = "ok7", fir.proc_attrs = #fir.proc_attrs<bind_c>} {
     entry s4() bind(c,name=foo//'7')
 ! CHECK: fir.call @ok2() {{.*}}: () -> ()
     call s1
diff --git a/flang/test/Parser/OpenMP/declare_target-device_type.f90 b/flang/test/Parser/OpenMP/declare_target-device_type.f90
index 0b4f75e7ddccb..40eb1c2fa4cae 100644
--- a/flang/test/Parser/OpenMP/declare_target-device_type.f90
+++ b/flang/test/Parser/OpenMP/declare_target-device_type.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=52 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=52 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 subroutine openmp_declare_target
     integer, save :: x, y
diff --git a/flang/test/Parser/OpenMP/in-reduction-clause.f90 b/flang/test/Parser/OpenMP/in-reduction-clause.f90
index 16f045771d732..776ead3824b71 100644
--- a/flang/test/Parser/OpenMP/in-reduction-clause.f90
+++ b/flang/test/Parser/OpenMP/in-reduction-clause.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 ! Check for IN_REDUCTION() clause on OpenMP constructs
 
diff --git a/flang/test/Parser/OpenMP/order-clause01.f90 b/flang/test/Parser/OpenMP/order-clause01.f90
index d7efaf0f67c23..41e131f9b5428 100644
--- a/flang/test/Parser/OpenMP/order-clause01.f90
+++ b/flang/test/Parser/OpenMP/order-clause01.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=50 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=50 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 ! Check for ORDER([order-modifier :]concurrent) clause on OpenMP constructs
 
diff --git a/flang/test/Parser/OpenMP/tile-size.f90 b/flang/test/Parser/OpenMP/tile-size.f90
index f40dc3819af07..64bc3c5319e88 100644
--- a/flang/test/Parser/OpenMP/tile-size.f90
+++ b/flang/test/Parser/OpenMP/tile-size.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 subroutine openmp_tiles(x)
 
diff --git a/flang/test/Parser/OpenMP/unroll-full.f90 b/flang/test/Parser/OpenMP/unroll-full.f90
index 3f26f61fc9aa0..30d2f46624991 100644
--- a/flang/test/Parser/OpenMP/unroll-full.f90
+++ b/flang/test/Parser/OpenMP/unroll-full.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=51 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 subroutine openmp_parse_unroll(x)
 
diff --git a/flang/test/Parser/OpenMP/unroll.f90 b/flang/test/Parser/OpenMP/unroll.f90
index 93163a3390db4..8ac2a74166773 100644
--- a/flang/test/Parser/OpenMP/unroll.f90
+++ b/flang/test/Parser/OpenMP/unroll.f90
@@ -1,5 +1,5 @@
-! RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp %s | FileCheck --ignore-case %s
-! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+! RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp -fopenmp-version=51 %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=51 %s | FileCheck --check-prefix="PARSE-TREE" %s
 
 subroutine openmp_parse_unroll(x)
 
diff --git a/flang/test/Semantics/OpenMP/atomic-hint-clause.f90 b/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
index e157b7e1e73a7..f724a69345f6e 100644
--- a/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
+++ b/flang/test/Semantics/OpenMP/atomic-hint-clause.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags 
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! Semantic checks on hint clauses, as they appear on atomic constructs
 
 program sample
diff --git a/flang/test/Semantics/OpenMP/atomic01.f90 b/flang/test/Semantics/OpenMP/atomic01.f90
index f0e1b47d2fa16..538db316f6e7f 100644
--- a/flang/test/Semantics/OpenMP/atomic01.f90
+++ b/flang/test/Semantics/OpenMP/atomic01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! Semantic checks for OpenMP 5.0 standard 2.17.7 atomic Construct.
 
 use omp_lib
diff --git a/flang/test/Semantics/OpenMP/atomic05.f90 b/flang/test/Semantics/OpenMP/atomic05.f90
index 2d9566463309c..f37aabcfce06e 100644
--- a/flang/test/Semantics/OpenMP/atomic05.f90
+++ b/flang/test/Semantics/OpenMP/atomic05.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags -fopenmp-version=50
 
 ! This tests the various semantics related to the clauses of various OpenMP atomic constructs
 
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 020d63f735596..24540492e7327 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags %openmp_module_flag
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags %openmp_module_flag -fopenmp-version=50
 use omp_lib
 ! Check OpenMP clause validity for the following directives:
 !
diff --git a/flang/test/Semantics/OpenMP/clause-validity02.f90 b/flang/test/Semantics/OpenMP/clause-validity02.f90
new file mode 100644
index 0000000000000..7e61bf0fd3177
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/clause-validity02.f90
@@ -0,0 +1,9 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=45
+
+subroutine bad_in_45(h_ptr)
+  integer, pointer :: h_ptr
+  !ERROR: USE_DEVICE_ADDR clause is not allowed on directive TARGET DATA in OpenMP v4.5, try -fopenmp-version=50
+  !$omp target data use_device_addr(h_ptr)
+  !$omp end target data
+end
+
diff --git a/flang/test/Semantics/OpenMP/declarative-directive.f90 b/flang/test/Semantics/OpenMP/declarative-directive.f90
index 4d10dc2d1b123..8d6762b87adb9 100644
--- a/flang/test/Semantics/OpenMP/declarative-directive.f90
+++ b/flang/test/Semantics/OpenMP/declarative-directive.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 
 ! Check OpenMP declarative directives
 
diff --git a/flang/test/Semantics/OpenMP/declare-target01.f90 b/flang/test/Semantics/OpenMP/declare-target01.f90
index 2c50a9248280b..0651d3b5d89c1 100644
--- a/flang/test/Semantics/OpenMP/declare-target01.f90
+++ b/flang/test/Semantics/OpenMP/declare-target01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=52
 ! OpenMP Version 5.1
 ! Check OpenMP construct validity for the following directives:
 ! 2.14.7 Declare Target Directive
diff --git a/flang/test/Semantics/OpenMP/declare-target02.f90 b/flang/test/Semantics/OpenMP/declare-target02.f90
index 8166e10d702b8..0f12180587f83 100644
--- a/flang/test/Semantics/OpenMP/declare-target02.f90
+++ b/flang/test/Semantics/OpenMP/declare-target02.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=52
 ! OpenMP Version 5.1
 ! Check OpenMP construct validity for the following directives:
 ! 2.14.7 Declare Target Directive
diff --git a/flang/test/Semantics/OpenMP/declare-target06.f90 b/flang/test/Semantics/OpenMP/declare-target06.f90
index a1c55d39e1b68..9abcfcecb681a 100644
--- a/flang/test/Semantics/OpenMP/declare-target06.f90
+++ b/flang/test/Semantics/OpenMP/declare-target06.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=52
 ! OpenMP Version 5.1
 ! Check OpenMP construct validity for the following directives:
 ! 2.14.7 Declare Target Directive
diff --git a/flang/test/Semantics/OpenMP/device-constructs.f90 b/flang/test/Semantics/OpenMP/device-constructs.f90
index 1ac00ef922c6b..4047fbf3fa811 100644
--- a/flang/test/Semantics/OpenMP/device-constructs.f90
+++ b/flang/test/Semantics/OpenMP/device-constructs.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=51
 ! Check OpenMP clause validity for the following directives:
 !     2.10 Device constructs
 program main
diff --git a/flang/test/Semantics/OpenMP/flush02.f90 b/flang/test/Semantics/OpenMP/flush02.f90
index d12c76bad0334..f06719f302fd7 100644
--- a/flang/test/Semantics/OpenMP/flush02.f90
+++ b/flang/test/Semantics/OpenMP/flush02.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 
 ! Check OpenMP 5.0 - 2.17.8 flush Construct
 ! Restriction -
diff --git a/flang/test/Semantics/OpenMP/if-clause.f90 b/flang/test/Semantics/OpenMP/if-clause.f90
index 493c6c873bfbf..7aeb617e53563 100644
--- a/flang/test/Semantics/OpenMP/if-clause.f90
+++ b/flang/test/Semantics/OpenMP/if-clause.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 ! Check OpenMP 'if' clause validity for all directives that can have it
 
 program main
diff --git a/flang/test/Semantics/OpenMP/nontemporal.f90 b/flang/test/Semantics/OpenMP/nontemporal.f90
index 6d24849575ee9..ad0ebc85b5ce7 100644
--- a/flang/test/Semantics/OpenMP/nontemporal.f90
+++ b/flang/test/Semantics/OpenMP/nontemporal.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s  %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s  %flang -fopenmp -fopenmp-version=50
 ! REQUIRES: shell
 ! Check OpenMP clause validity for NONTEMPORAL clause
 
diff --git a/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90 b/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
index 0d4da5485af04..9701c1db92c1c 100644
--- a/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
+++ b/flang/test/Semantics/OpenMP/omp-atomic-assignment-stmt.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=50
 ! Semantic checks for various assignments related to atomic constructs
 
 program sample
diff --git a/flang/test/Semantics/OpenMP/order-clause01.f90 b/flang/test/Semantics/OpenMP/order-clause01.f90
index 247791fac15b4..bb6e19e4ddd14 100644
--- a/flang/test/Semantics/OpenMP/order-clause01.f90
+++ b/flang/test/Semantics/OpenMP/order-clause01.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=50
 
 subroutine omp_order()
  integer :: i, j = 1
diff --git a/flang/test/Semantics/OpenMP/requires-atomic01.f90 b/flang/test/Semantics/OpenMP/requires-atomic01.f90
index cb7b1bc1ac52a..ae9fd086015dd 100644
--- a/flang/test/Semantics/OpenMP/requires-atomic01.f90
+++ b/flang/test/Semantics/OpenMP/requires-atomic01.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=50 -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
 ! Ensure that requires atomic_default_mem_order is used to update atomic
 ! operations with no explicit memory order set.
 program requires
diff --git a/flang/test/Semantics/OpenMP/requires-atomic02.f90 b/flang/test/Semantics/OpenMP/requires-atomic02.f90
index 5a4249794f7b5..4976a9667eb78 100644
--- a/flang/test/Semantics/OpenMP/requires-atomic02.f90
+++ b/flang/test/Semantics/OpenMP/requires-atomic02.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=50 -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
 ! Ensure that requires atomic_default_mem_order is used to update atomic
 ! operations with no explicit memory order set. ACQ_REL clause tested here.
 program requires
diff --git a/flang/test/Semantics/OpenMP/requires04.f90 b/flang/test/Semantics/OpenMP/requires04.f90
index bb4101c1cbd6c..1fbb3aa6219bc 100644
--- a/flang/test/Semantics/OpenMP/requires04.f90
+++ b/flang/test/Semantics/OpenMP/requires04.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 ! OpenMP Version 5.0
 ! 2.4 Requires directive
 ! Target-related clauses in 'requires' directives must come strictly before any
diff --git a/flang/test/Semantics/OpenMP/requires05.f90 b/flang/test/Semantics/OpenMP/requires05.f90
index dd27e3895e394..f410f0104d899 100644
--- a/flang/test/Semantics/OpenMP/requires05.f90
+++ b/flang/test/Semantics/OpenMP/requires05.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=52
 ! OpenMP Version 5.0
 ! 2.4 Requires directive
 ! Target-related clauses in 'requires' directives must come strictly before any
diff --git a/flang/test/Semantics/OpenMP/simd-nontemporal.f90 b/flang/test/Semantics/OpenMP/simd-nontemporal.f90
index a488edd98cdc3..42518fd1aabe1 100644
--- a/flang/test/Semantics/OpenMP/simd-nontemporal.f90
+++ b/flang/test/Semantics/OpenMP/simd-nontemporal.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=50
 
 ! OpenMP Version 4.5
 ! 2.8.1 simd Construct
diff --git a/flang/test/Semantics/OpenMP/target01.f90 b/flang/test/Semantics/OpenMP/target01.f90
index 9836f0112738f..545cc8a7b69f8 100644
--- a/flang/test/Semantics/OpenMP/target01.f90
+++ b/flang/test/Semantics/OpenMP/target01.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=51
 
 subroutine foo(b)
 use iso_c_binding
diff --git a/flang/test/Semantics/OpenMP/taskgroup01.f90 b/flang/test/Semantics/OpenMP/taskgroup01.f90
index 98c9aabffa234..e05051387411a 100644
--- a/flang/test/Semantics/OpenMP/taskgroup01.f90
+++ b/flang/test/Semantics/OpenMP/taskgroup01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang %openmp_flags -fopenmp-version=50
 
 use omp_lib
   implicit none
diff --git a/flang/test/Semantics/OpenMP/use_device_addr.f90 b/flang/test/Semantics/OpenMP/use_device_addr.f90
index 93a7643b5eb48..0571a1fc06f81 100644
--- a/flang/test/Semantics/OpenMP/use_device_addr.f90
+++ b/flang/test/Semantics/OpenMP/use_device_addr.f90
@@ -1,4 +1,4 @@
-! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-version=51 -fdebug-dump-symbols %s | FileCheck %s
 ! OpenMP Version 5.1
 ! 2.14.2 use_device_addr clause
 ! List item that appears in a use_device_addr clause has corresponding storage
diff --git a/flang/test/Semantics/OpenMP/use_device_addr1.f90 b/flang/test/Semantics/OpenMP/use_device_addr1.f90
index 867e324b68ad9..e6a3e6e5b2a2d 100644
--- a/flang/test/Semantics/OpenMP/use_device_addr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_addr1.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=50
 ! OpenMP Version 5.0
 ! 2.10.1 use_device_ptr clause
 ! List item in USE_DEVICE_ADDR clause must not be structure element.
diff --git a/flang/test/Semantics/OpenMP/use_device_ptr1.f90 b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
index 176fb5f35a849..41dbadc59ce7c 100644
--- a/flang/test/Semantics/OpenMP/use_device_ptr1.f90
+++ b/flang/test/Semantics/OpenMP/use_device_ptr1.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -fopenmp-version=50
 ! OpenMP Version 5.0
 ! 2.10.1 use_device_ptr clause
 ! List item in USE_DEVICE_PTR clause must not be structure element.
diff --git a/flang/test/Transforms/debug-107988.fir b/flang/test/Transforms/debug-107988.fir
index 308f78a865120..0b08cf1c0b2eb 100644
--- a/flang/test/Transforms/debug-107988.fir
+++ b/flang/test/Transforms/debug-107988.fir
@@ -17,7 +17,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
 // CHECK: func.func @test
 // CHECK: %[[V1:.*]]:2 = fir.unboxchar{{.*}}
 // CHECK: %[[V2:.*]] = fir.convert %[[V1]]#1 : (index) -> i64
-// CHECK: llvm.intr.dbg.value #di_local_variable = %[[V2]] : i64
+// CHECK: llvm.intr.dbg.value #[[VAR]] = %[[V2]] : i64
 // CHECK: #[[STR_TY:.*]] = #llvm.di_string_type<tag = DW_TAG_string_type, name = "", stringLength = #[[VAR]], encoding = DW_ATE_ASCII>
 // CHECK: #llvm.di_local_variable<{{.*}}name = "str"{{.*}}type = #[[STR_TY]]>
 
diff --git a/flang/test/Transforms/debug-variable-char-len.fir b/flang/test/Transforms/debug-variable-char-len.fir
new file mode 100644
index 0000000000000..598d97cee970a
--- /dev/null
+++ b/flang/test/Transforms/debug-variable-char-len.fir
@@ -0,0 +1,31 @@
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s -o - | FileCheck %s
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<>} {
+  func.func @foo(%arg0: !fir.ref<!fir.char<1,?>> {fir.bindc_name = "str1"} , %arg1: !fir.ref<i64> {fir.bindc_name = "len1"} loc("/home/haqadeer/work/fortran/t1/../str.f90":1:1), %arg2: i64) {
+    %0 = fir.emboxchar %arg0, %arg2 : (!fir.ref<!fir.char<1,?>>, i64) -> !fir.boxchar<1>
+    %c4_i32 = arith.constant 4 : i32
+    %c6_i32 = arith.constant 6 : i32
+    %c0_i64 = arith.constant 0 : i64
+    %1 = fir.undefined !fir.dscope
+    %2 = fircg.ext_declare %arg1 dummy_scope %1 {uniq_name = "_QFfooElen1"} : (!fir.ref<i64>, !fir.dscope) -> !fir.ref<i64> loc(#loc1)
+    %3:2 = fir.unboxchar %0 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+    %4 = fir.load %2 : !fir.ref<i64>
+    %5 = arith.cmpi sgt, %4, %c0_i64 : i64
+    %6 = arith.select %5, %4, %c0_i64 : i64
+    %7 = fircg.ext_declare %3#0 typeparams %6 dummy_scope %1 {uniq_name = "_QFfooEstr1"} : (!fir.ref<!fir.char<1,?>>, i64, !fir.dscope) -> !fir.ref<!fir.char<1,?>> loc(#loc2)
+    return
+  } loc(#loc3)
+}
+
+
+#loc1 = loc("test.f90":18:1)
+#loc2 = loc("test.f90":17:1)
+#loc3 = loc("test.f90":15:1)
+
+// CHECK: #[[VAR:.*]] = #llvm.di_local_variable<{{.*}}name = "._QFfooEstr1"{{.*}}flags = Artificial>
+// CHECK: func.func @foo
+// CHECK: llvm.intr.dbg.value #[[VAR]]
+// CHECK: return
+// CHECK: #[[STR_TY:.*]] = #llvm.di_string_type<tag = DW_TAG_string_type, name = "", stringLength = #[[VAR]], encoding = DW_ATE_ASCII>
+// CHECK: #llvm.di_local_variable<{{.*}}name = "str1"{{.*}}type = #[[STR_TY]]>
+
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index ac3de35319106..0a008d577cc25 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -15,6 +15,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Common/Fortran-features.h"
+#include "flang/Common/LangOptions.h"
 #include "flang/Common/OpenMP-features.h"
 #include "flang/Common/Version.h"
 #include "flang/Common/default-kinds.h"
@@ -508,6 +509,21 @@ int main(int argc, char **argv) {
   options.predefinitions.emplace_back(
       "__flang_patchlevel__"s, std::string{FLANG_VERSION_PATCHLEVEL_STRING});
 
+  Fortran::common::LangOptions langOpts;
+  langOpts.NoGPULib = setNoGPULib;
+  langOpts.OpenMPVersion = setOpenMPVersion;
+  langOpts.OpenMPIsTargetDevice = enableOpenMPDevice;
+  langOpts.OpenMPIsGPU = enableOpenMPGPU;
+  langOpts.OpenMPForceUSM = enableOpenMPForceUSM;
+  langOpts.OpenMPTargetDebug = setOpenMPTargetDebug;
+  langOpts.OpenMPThreadSubscription = setOpenMPThreadSubscription;
+  langOpts.OpenMPTeamSubscription = setOpenMPTeamSubscription;
+  langOpts.OpenMPNoThreadState = setOpenMPNoThreadState;
+  langOpts.OpenMPNoNestedParallelism = setOpenMPNoNestedParallelism;
+  std::transform(targetTriplesOpenMP.begin(), targetTriplesOpenMP.end(),
+                 std::back_inserter(langOpts.OMPTargetTriples),
+                 [](const std::string &str) { return llvm::Triple(str); });
+
   // enable parsing of OpenMP
   if (enableOpenMP) {
     options.features.Enable(Fortran::common::LanguageFeature::OpenMP);
@@ -539,7 +555,7 @@ int main(int argc, char **argv) {
   Fortran::parser::AllSources allSources;
   Fortran::parser::AllCookedSources allCookedSources(allSources);
   Fortran::semantics::SemanticsContext semanticsContext{
-      defaultKinds, options.features, allCookedSources};
+      defaultKinds, options.features, langOpts, allCookedSources};
   semanticsContext.set_moduleDirectory(moduleDir)
       .set_moduleFileSuffix(moduleSuffix)
       .set_searchDirectories(includeDirs)
diff --git a/libc/hdr/errno_macros.h b/libc/hdr/errno_macros.h
index 198b5233d4409..27ea49977d8c8 100644
--- a/libc/hdr/errno_macros.h
+++ b/libc/hdr/errno_macros.h
@@ -14,9 +14,9 @@
 #ifdef __linux__
 #include <linux/errno.h>
 
-#include "llvm-libc-macros/error-number-macros.h"
+#include "include/llvm-libc-macros/error-number-macros.h"
 #else // __linux__
-#include "llvm-libc-macros/generic-error-number-macros.h"
+#include "include/llvm-libc-macros/generic-error-number-macros.h"
 #endif
 
 #else // Overlay mode
diff --git a/libc/src/stdio/printf_core/parser.h b/libc/src/stdio/printf_core/parser.h
index e2cb734b5be71..acbbaa25b1c9d 100644
--- a/libc/src/stdio/printf_core/parser.h
+++ b/libc/src/stdio/printf_core/parser.h
@@ -265,7 +265,7 @@ template <typename ArgProvider> class Parser {
       case ('m'):
         // %m is an odd conversion in that it doesn't consume an argument, it
         // just takes the current value of errno as its argument.
-        section.conv_val_raw = libc_errno;
+        section.conv_val_raw = static_cast<int>(libc_errno);
         break;
 #endif // LIBC_COPT_PRINTF_DISABLE_STRERROR
 #ifndef LIBC_COPT_PRINTF_DISABLE_WRITE_INT
diff --git a/libc/test/src/stdio/fgets_test.cpp b/libc/test/src/stdio/fgets_test.cpp
index d005a71710d21..39337262f1e00 100644
--- a/libc/test/src/stdio/fgets_test.cpp
+++ b/libc/test/src/stdio/fgets_test.cpp
@@ -43,6 +43,8 @@ TEST(LlvmLibcFgetsTest, WriteAndReadCharacters) {
   file = LIBC_NAMESPACE::fopen(FILENAME, "r");
   ASSERT_FALSE(file == nullptr);
 
+  // The GPU build relies on the host C library, so this check may be different.
+#ifndef LIBC_TARGET_ARCH_IS_GPU
   // If we request just 1 byte, it should return just a null byte and not
   // advance the read head. This is implementation defined.
   output = LIBC_NAMESPACE::fgets(buff, 1, file);
@@ -54,6 +56,7 @@ TEST(LlvmLibcFgetsTest, WriteAndReadCharacters) {
   // This is also implementation defined.
   output = LIBC_NAMESPACE::fgets(buff, 0, file);
   ASSERT_TRUE(output == nullptr);
+#endif
 
   const char *output_arr[] = {
       "1234567", "89\n", "1234567", "\n", "123456\n", "1",
diff --git a/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp b/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
index 1beef8170475a..9ed9d99a60724 100644
--- a/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
+++ b/libc/utils/gpu/loader/amdgpu/amdhsa-loader.cpp
@@ -231,8 +231,8 @@ hsa_status_t launch_kernel(hsa_agent_t dev_agent, hsa_executable_t executable,
   std::memcpy(args, &kernel_args, sizeof(args_t));
 
   // Initialize the necessary implicit arguments to the proper values.
-  bool dims = 1 + (params.num_blocks_y * params.num_threads_y != 1) +
-              (params.num_blocks_z * params.num_threads_z != 1);
+  int dims = 1 + (params.num_blocks_y * params.num_threads_y != 1) +
+             (params.num_blocks_z * params.num_threads_z != 1);
   implicit_args_t *implicit_args = reinterpret_cast<implicit_args_t *>(
       reinterpret_cast<uint8_t *>(args) + sizeof(args_t));
   implicit_args->grid_dims = dims;
diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp
index aa65dfe69c385..602b266f9dacb 100644
--- a/libc/utils/gpu/server/rpc_server.cpp
+++ b/libc/utils/gpu/server/rpc_server.cpp
@@ -285,6 +285,7 @@ rpc_status_t handle_server_impl(
       FILE *file = fopen(reinterpret_cast<char *>(paths[id]),
                          reinterpret_cast<char *>(buffer->data));
       buffer->data[0] = reinterpret_cast<uintptr_t>(file);
+      delete[] reinterpret_cast<uint8_t *>(paths[id]);
     });
     break;
   }
diff --git a/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake b/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
index c0f2bad1c95af..f63436c767947 100644
--- a/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
+++ b/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
@@ -1,2 +1,2 @@
 set(LIBCXX_HARDENING_MODE "fast" CACHE STRING "")
-set(LIBCXX_ABI_DEFINES "_LIBCPP_ABI_BOUNDED_ITERATORS;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR" CACHE STRING "")
+set(LIBCXX_ABI_DEFINES "_LIBCPP_ABI_BOUNDED_ITERATORS;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING;_LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR;_LIBCPP_ABI_BOUNDED_UNIQUE_PTR" CACHE STRING "")
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index bde203ea12f14..a62c4992020a0 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -50,7 +50,7 @@
 "`LWG4013 <https://wg21.link/LWG4013>`__","``lazy_split_view::outer-iterator::value_type`` should not provide default constructor","2024-03 (Tokyo)","","",""
 "`LWG4016 <https://wg21.link/LWG4016>`__","container-insertable checks do not match what container-inserter does","2024-03 (Tokyo)","","",""
 "`LWG4023 <https://wg21.link/LWG4023>`__","Preconditions of ``std::basic_streambuf::setg/setp``","2024-03 (Tokyo)","|Complete|","19.0",""
-"`LWG4025 <https://wg21.link/LWG4025>`__","Move assignment operator of ``std::expected<cv void, E>`` should not be conditionally deleted","2024-03 (Tokyo)","","",""
+"`LWG4025 <https://wg21.link/LWG4025>`__","Move assignment operator of ``std::expected<cv void, E>`` should not be conditionally deleted","2024-03 (Tokyo)","|Complete|","20.0",""
 "`LWG4030 <https://wg21.link/LWG4030>`__","Clarify whether arithmetic expressions in ``[numeric.sat.func]`` are mathematical or C++","2024-03 (Tokyo)","|Nothing To Do|","",""
 "`LWG4031 <https://wg21.link/LWG4031>`__","``bad_expected_access<void>`` member functions should be ``noexcept``","2024-03 (Tokyo)","|Complete|","16.0",""
 "`LWG4035 <https://wg21.link/LWG4035>`__","``single_view`` should provide ``empty``","2024-03 (Tokyo)","|Complete|","19.0",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 8c61009167ddc..8a63280053340 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -23,7 +23,6 @@ set(files
   __algorithm/find_if.h
   __algorithm/find_if_not.h
   __algorithm/find_segment_if.h
-  __algorithm/fold.h
   __algorithm/for_each.h
   __algorithm/for_each_n.h
   __algorithm/for_each_segment.h
@@ -98,6 +97,7 @@ set(files
   __algorithm/ranges_find_if.h
   __algorithm/ranges_find_if_not.h
   __algorithm/ranges_find_last.h
+  __algorithm/ranges_fold.h
   __algorithm/ranges_for_each.h
   __algorithm/ranges_for_each_n.h
   __algorithm/ranges_generate.h
@@ -536,6 +536,7 @@ set(files
   __memory/allocator_arg_t.h
   __memory/allocator_destructor.h
   __memory/allocator_traits.h
+  __memory/array_cookie.h
   __memory/assume_aligned.h
   __memory/auto_ptr.h
   __memory/builtin_new_allocator.h
@@ -687,7 +688,6 @@ set(files
   __ranges/views.h
   __ranges/zip_view.h
   __split_buffer
-  __std_clang_module
   __std_mbstate_t.h
   __stop_token/atomic_unique_lock.h
   __stop_token/intrusive_list_view.h
diff --git a/libcxx/include/__algorithm/fold.h b/libcxx/include/__algorithm/ranges_fold.h
similarity index 97%
rename from libcxx/include/__algorithm/fold.h
rename to libcxx/include/__algorithm/ranges_fold.h
index 1bcb3be9aadab..d2c3921398504 100644
--- a/libcxx/include/__algorithm/fold.h
+++ b/libcxx/include/__algorithm/ranges_fold.h
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _LIBCPP___ALGORITHM_FOLD_H
-#define _LIBCPP___ALGORITHM_FOLD_H
+#ifndef _LIBCPP___ALGORITHM_RANGES_FOLD_H
+#define _LIBCPP___ALGORITHM_RANGES_FOLD_H
 
 #include <__concepts/assignable.h>
 #include <__concepts/constructible.h>
@@ -126,4 +126,4 @@ _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
 
-#endif // _LIBCPP___ALGORITHM_FOLD_H
+#endif // _LIBCPP___ALGORITHM_RANGES_FOLD_H
diff --git a/libcxx/include/__config b/libcxx/include/__config
index f90c966aad695..0d71264611ff6 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -649,6 +649,10 @@ typedef __char32_t char32_t;
          __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500)
 #      define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
 #    endif
+#    if (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) &&                                                    \
+         __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 130000)
+#      define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
+#    endif
 #  elif defined(__ANDROID__) && __ANDROID_API__ < 28
 // Android only provides aligned_alloc when targeting API 28 or higher.
 #    define _LIBCPP_HAS_NO_C11_ALIGNED_ALLOC
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index 707e10b5ceb53..62c129f5921de 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -181,6 +181,13 @@
 #  define _LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING
 #endif
 
+// Tracks the bounds of the array owned by std::unique_ptr<T[]>, allowing it to trap when accessed out-of-bounds.
+// Note that limited bounds checking is also available outside of this ABI configuration, but only some categories
+// of types can be checked.
+//
+// ABI impact: This causes the layout of std::unique_ptr<T[]> to change and its size to increase.
+// #define _LIBCPP_ABI_BOUNDED_UNIQUE_PTR
+
 #if defined(_LIBCPP_COMPILER_CLANG_BASED)
 #  if defined(__APPLE__)
 #    if defined(__i386__) || defined(__x86_64__)
diff --git a/libcxx/include/__configuration/availability.h b/libcxx/include/__configuration/availability.h
index ab483a07c9c13..b10f29590a2c9 100644
--- a/libcxx/include/__configuration/availability.h
+++ b/libcxx/include/__configuration/availability.h
@@ -138,15 +138,31 @@
 #  define _LIBCPP_INTRODUCED_IN_LLVM_19_ATTRIBUTE __attribute__((unavailable))
 
 // LLVM 18
-// TODO: Fill this in
-#  define _LIBCPP_INTRODUCED_IN_LLVM_18 0
-#  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE __attribute__((unavailable))
+#  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 150000) ||       \
+      (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 180000) ||     \
+      (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 180000) ||             \
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 110000) ||       \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 90000) ||      \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 240000)
+#    define _LIBCPP_INTRODUCED_IN_LLVM_18 0
+#  else
+#    define _LIBCPP_INTRODUCED_IN_LLVM_18 1
+#  endif
+#  define _LIBCPP_INTRODUCED_IN_LLVM_18_ATTRIBUTE                                                                 \
+    __attribute__((availability(macos, strict, introduced = 15.0)))                                               \
+    __attribute__((availability(ios, strict, introduced = 18.0)))                                                 \
+    __attribute__((availability(tvos, strict, introduced = 18.0)))                                                \
+    __attribute__((availability(watchos, strict, introduced = 11.0)))                                             \
+    __attribute__((availability(bridgeos, strict, introduced = 9.0)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 24.0)))
 
 // LLVM 17
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140400) ||       \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170400) ||     \
       (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 170400) ||             \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100400)
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100400) ||       \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 80400) ||      \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 230400)
 #    define _LIBCPP_INTRODUCED_IN_LLVM_17 0
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_17 1
@@ -155,13 +171,17 @@
     __attribute__((availability(macos, strict, introduced = 14.4)))                                               \
     __attribute__((availability(ios, strict, introduced = 17.4)))                                                 \
     __attribute__((availability(tvos, strict, introduced = 17.4)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 10.4)))
+    __attribute__((availability(watchos, strict, introduced = 10.4)))                                             \
+    __attribute__((availability(bridgeos, strict, introduced = 8.4)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 23.4)))
 
 // LLVM 16
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 140000) ||       \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 170000) ||     \
       (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 170000) ||             \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100000)
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 100000) ||       \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 80000) ||      \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 230000)
 #    define _LIBCPP_INTRODUCED_IN_LLVM_16 0
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_16 1
@@ -170,13 +190,17 @@
     __attribute__((availability(macos, strict, introduced = 14.0)))                                               \
     __attribute__((availability(ios, strict, introduced = 17.0)))                                                 \
     __attribute__((availability(tvos, strict, introduced = 17.0)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 10.0)))
+    __attribute__((availability(watchos, strict, introduced = 10.0)))                                             \
+    __attribute__((availability(bridgeos, strict, introduced = 8.0)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 23.0)))
 
 // LLVM 15
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 130400) ||   \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 160500) || \
       (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 160500) ||         \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90500)
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90500) ||    \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 70500) ||  \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 220400)
 #    define _LIBCPP_INTRODUCED_IN_LLVM_15 0
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_15 1
@@ -185,7 +209,9 @@
     __attribute__((availability(macos, strict, introduced = 13.4)))                                               \
     __attribute__((availability(ios, strict, introduced = 16.5)))                                                 \
     __attribute__((availability(tvos, strict, introduced = 16.5)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 9.5)))
+    __attribute__((availability(watchos, strict, introduced = 9.5)))                                              \
+    __attribute__((availability(bridgeos, strict, introduced = 7.5)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 22.4)))
 
 // LLVM 14
 #  define _LIBCPP_INTRODUCED_IN_LLVM_14 _LIBCPP_INTRODUCED_IN_LLVM_15
@@ -195,7 +221,9 @@
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 130000) ||   \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 160000) || \
       (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 160000) ||         \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90000)
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 90000) ||    \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 70000) ||  \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 220000)
 #    define _LIBCPP_INTRODUCED_IN_LLVM_13 0
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_13 1
@@ -204,13 +232,17 @@
     __attribute__((availability(macos, strict, introduced = 13.0)))                                               \
     __attribute__((availability(ios, strict, introduced = 16.0)))                                                 \
     __attribute__((availability(tvos, strict, introduced = 16.0)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 9.0)))
+    __attribute__((availability(watchos, strict, introduced = 9.0)))                                              \
+    __attribute__((availability(bridgeos, strict, introduced = 7.0)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 22.0)))
 
 // LLVM 12
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 120300)   ||     \
       (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 150300) ||     \
       (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 150300)         ||     \
-      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 80300)
+      (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 80300)    ||     \
+      (defined(__ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_BRIDGE_OS_VERSION_MIN_REQUIRED__ < 60000)  ||     \
+      (defined(__ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_DRIVERKIT_VERSION_MIN_REQUIRED__ < 210300)
 #    define _LIBCPP_INTRODUCED_IN_LLVM_12 0
 #  else
 #    define _LIBCPP_INTRODUCED_IN_LLVM_12 1
@@ -219,7 +251,9 @@
     __attribute__((availability(macos, strict, introduced = 12.3)))                                               \
     __attribute__((availability(ios, strict, introduced = 15.3)))                                                 \
     __attribute__((availability(tvos, strict, introduced = 15.3)))                                                \
-    __attribute__((availability(watchos, strict, introduced = 8.3)))
+    __attribute__((availability(watchos, strict, introduced = 8.3)))                                              \
+    __attribute__((availability(bridgeos, strict, introduced = 6.0)))                                             \
+    __attribute__((availability(driverkit, strict, introduced = 21.3)))
 
 // LLVM 11
 #  if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000) ||   \
diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h
index f4ad455a19ea5..e04e17d1ebe69 100644
--- a/libcxx/include/__expected/expected.h
+++ b/libcxx/include/__expected/expected.h
@@ -1493,8 +1493,6 @@ class expected<_Tp, _Err> : private __expected_void_base<_Err> {
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI constexpr expected& operator=(expected&&) = delete;
-
   _LIBCPP_HIDE_FROM_ABI constexpr expected&
   operator=(expected&& __rhs) noexcept(is_nothrow_move_assignable_v<_Err> && is_nothrow_move_constructible_v<_Err>)
     requires(is_move_assignable_v<_Err> && is_move_constructible_v<_Err>)
diff --git a/libcxx/include/__format/formatter_integral.h b/libcxx/include/__format/formatter_integral.h
index beed3ab8d93df..0c04cce855a08 100644
--- a/libcxx/include/__format/formatter_integral.h
+++ b/libcxx/include/__format/formatter_integral.h
@@ -27,6 +27,7 @@
 #include <__type_traits/make_unsigned.h>
 #include <__utility/unreachable.h>
 #include <array>
+#include <cstdint>
 #include <limits>
 #include <string>
 #include <string_view>
diff --git a/libcxx/include/__memory/array_cookie.h b/libcxx/include/__memory/array_cookie.h
new file mode 100644
index 0000000000000..34eec64320610
--- /dev/null
+++ b/libcxx/include/__memory/array_cookie.h
@@ -0,0 +1,55 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___MEMORY_ARRAY_COOKIE_H
+#define _LIBCPP___MEMORY_ARRAY_COOKIE_H
+
+#include <__config>
+#include <__configuration/abi.h>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_trivially_destructible.h>
+#include <__type_traits/negation.h>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// Trait representing whether a type requires an array cookie at the start of its allocation when
+// allocated as `new T[n]` and deallocated as `delete array`.
+//
+// Under the Itanium C++ ABI [1], we know that an array cookie is available unless `T` is trivially
+// destructible and the call to `operator delete[]` is not a sized operator delete. Under ABIs other
+// than the Itanium ABI, we assume there are no array cookies.
+//
+// [1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-cookies
+#ifdef _LIBCPP_ABI_ITANIUM
+// TODO: Use a builtin instead
+// TODO: We should factor in the choice of the usual deallocation function in this determination.
+template <class _Tp>
+struct __has_array_cookie : _Not<is_trivially_destructible<_Tp> > {};
+#else
+template <class _Tp>
+struct __has_array_cookie : false_type {};
+#endif
+
+template <class _Tp>
+// Avoid failures when -fsanitize-address-poison-custom-array-cookie is enabled
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") size_t __get_array_cookie(_Tp const* __ptr) {
+  static_assert(
+      __has_array_cookie<_Tp>::value, "Trying to access the array cookie of a type that is not guaranteed to have one");
+  size_t const* __cookie = reinterpret_cast<size_t const*>(__ptr) - 1; // TODO: Use a builtin instead
+  return *__cookie;
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___MEMORY_ARRAY_COOKIE_H
diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h
index 9ca13d0e4fd1a..11215dc111e36 100644
--- a/libcxx/include/__memory/unique_ptr.h
+++ b/libcxx/include/__memory/unique_ptr.h
@@ -10,6 +10,7 @@
 #ifndef _LIBCPP___MEMORY_UNIQUE_PTR_H
 #define _LIBCPP___MEMORY_UNIQUE_PTR_H
 
+#include <__assert>
 #include <__compare/compare_three_way.h>
 #include <__compare/compare_three_way_result.h>
 #include <__compare/three_way_comparable.h>
@@ -17,8 +18,10 @@
 #include <__functional/hash.h>
 #include <__functional/operations.h>
 #include <__memory/allocator_traits.h> // __pointer
+#include <__memory/array_cookie.h>
 #include <__memory/auto_ptr.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/pointer_traits.h>
 #include <__type_traits/add_lvalue_reference.h>
 #include <__type_traits/common_type.h>
 #include <__type_traits/conditional.h>
@@ -27,6 +30,7 @@
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_array.h>
 #include <__type_traits/is_assignable.h>
+#include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/is_constructible.h>
 #include <__type_traits/is_convertible.h>
 #include <__type_traits/is_function.h>
@@ -41,7 +45,9 @@
 #include <__utility/declval.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
+#include <__utility/private_constructor_tag.h>
 #include <cstddef>
+#include <cstdint>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -292,6 +298,91 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr {
   }
 };
 
+// Bounds checking in unique_ptr<T[]>
+// ==================================
+//
+// We provide some helper classes that allow bounds checking when accessing a unique_ptr<T[]>.
+// There are a few cases where bounds checking can be implemented:
+//
+// 1. When an array cookie (see [1]) exists at the beginning of the array allocation, we are
+//    able to reuse that cookie to extract the size of the array and perform bounds checking.
+//    An array cookie is a size inserted at the beginning of the allocation by the compiler.
+//    That size is inserted implicitly when doing `new T[n]` in some cases, and its purpose
+//    is to allow the runtime to destroy the `n` array elements when doing `delete array`.
+//    When we are able to use array cookies, we reuse information already available in the
+//    current runtime, so bounds checking does not require changing libc++'s ABI.
+//
+// 2. When the "bounded unique_ptr" ABI configuration (controlled by `_LIBCPP_ABI_BOUNDED_UNIQUE_PTR`)
+//    is enabled, we store the size of the allocation (when it is known) so we can check it when
+//    indexing into the `unique_ptr`. That changes the layout of `std::unique_ptr<T[]>`, which is
+//    an ABI break from the default configuration.
+//
+//    Note that even under this ABI configuration, we can't always know the size of the unique_ptr.
+//    Indeed, the size of the allocation can only be known when the unique_ptr is created via
+//    make_unique or a similar API. For example, it can't be known when constructed from an arbitrary
+//    pointer, in which case we are not able to check the bounds on access:
+//
+//      unique_ptr<T[], MyDeleter> ptr(new T[3]);
+//
+//    When we don't know the size of the allocation via the API used to create the unique_ptr, we
+//    try to fall back to using an array cookie when available.
+//
+//    Finally, note that when this ABI configuration is enabled, we have no choice but to always
+//    make space for a size to be stored in the unique_ptr. Indeed, while we might want to avoid
+//    storing the size when an array cookie is available, knowing whether an array cookie is available
+//    requires the type stored in the unique_ptr to be complete, while unique_ptr can normally
+//    accommodate incomplete types.
+//
+// (1) Implementation where we rely on the array cookie to know the size of the allocation, if
+//     an array cookie exists.
+struct __unique_ptr_array_bounds_stateless {
+  __unique_ptr_array_bounds_stateless() = default;
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __unique_ptr_array_bounds_stateless(size_t) {}
+
+  template <class _Tp, __enable_if_t<__has_array_cookie<_Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp* __ptr, size_t __index) const {
+    // In constant expressions, we can't check the array cookie so we just pretend that the index
+    // is in-bounds. The compiler catches invalid accesses anyway.
+    if (__libcpp_is_constant_evaluated())
+      return true;
+    size_t __cookie = std::__get_array_cookie(__ptr);
+    return __index < __cookie;
+  }
+
+  template <class _Tp, __enable_if_t<!__has_array_cookie<_Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp*, size_t) const {
+    return true; // If we don't have an array cookie, we assume the access is in-bounds
+  }
+};
+
+// (2) Implementation where we store the size in the class whenever we have it.
+//
+// Semantically, we'd need to store the size as an optional<size_t>. However, since that
+// is really heavy weight, we instead store a size_t and use SIZE_MAX as a magic value
+// meaning that we don't know the size.
+struct __unique_ptr_array_bounds_stored {
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __unique_ptr_array_bounds_stored() : __size_(SIZE_MAX) {}
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __unique_ptr_array_bounds_stored(size_t __size) : __size_(__size) {}
+
+  // Use the array cookie if there's one
+  template <class _Tp, __enable_if_t<__has_array_cookie<_Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp* __ptr, size_t __index) const {
+    if (__libcpp_is_constant_evaluated())
+      return true;
+    size_t __cookie = std::__get_array_cookie(__ptr);
+    return __index < __cookie;
+  }
+
+  // Otherwise, fall back on the stored size (if any)
+  template <class _Tp, __enable_if_t<!__has_array_cookie<_Tp>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR bool __in_bounds(_Tp*, size_t __index) const {
+    return __index < __size_;
+  }
+
+private:
+  size_t __size_;
+};
+
 template <class _Tp, class _Dp>
 class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> {
 public:
@@ -300,8 +391,9 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   typedef typename __pointer<_Tp, deleter_type>::type pointer;
 
   // A unique_ptr contains the following members which may be trivially relocatable:
-  // - pointer : this may be trivially relocatable, so it's checked
+  // - pointer: this may be trivially relocatable, so it's checked
   // - deleter_type: this may be trivially relocatable, so it's checked
+  // - (optionally) size: this is trivially relocatable
   //
   // This unique_ptr implementation only contains a pointer to the unique object and a deleter, so there are no
   // references to itself. This means that the entire structure is trivially relocatable if its members are.
@@ -311,7 +403,16 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
       void>;
 
 private:
+  template <class _Up, class _OtherDeleter>
+  friend class unique_ptr;
+
   _LIBCPP_COMPRESSED_PAIR(pointer, __ptr_, deleter_type, __deleter_);
+#ifdef _LIBCPP_ABI_BOUNDED_UNIQUE_PTR
+  using _BoundsChecker = __unique_ptr_array_bounds_stored;
+#else
+  using _BoundsChecker = __unique_ptr_array_bounds_stateless;
+#endif
+  _LIBCPP_NO_UNIQUE_ADDRESS _BoundsChecker __checker_;
 
   template <class _From>
   struct _CheckArrayPointerConversion : is_same<_From, pointer> {};
@@ -373,6 +474,12 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
       : __ptr_(__p),
         __deleter_() {}
 
+  // Private constructor used by make_unique & friends to pass the size that was allocated
+  template <class _Tag, class _Ptr, __enable_if_t<is_same<_Tag, __private_constructor_tag>::value, int> = 0>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 explicit unique_ptr(_Tag, _Ptr __ptr, size_t __size) _NOEXCEPT
+      : __ptr_(__ptr),
+        __checker_(__size) {}
+
   template <class _Pp,
             bool _Dummy = true,
             class       = _EnableIfDeleterConstructible<_LValRefType<_Dummy> >,
@@ -411,11 +518,13 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr&& __u) _NOEXCEPT
       : __ptr_(__u.release()),
-        __deleter_(std::forward<deleter_type>(__u.get_deleter())) {}
+        __deleter_(std::forward<deleter_type>(__u.get_deleter())),
+        __checker_(std::move(__u.__checker_)) {}
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr&& __u) _NOEXCEPT {
     reset(__u.release());
     __deleter_ = std::forward<deleter_type>(__u.get_deleter());
+    __checker_ = std::move(std::move(__u.__checker_));
     return *this;
   }
 
@@ -425,7 +534,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
             class = _EnableIfDeleterConvertible<_Ep> >
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT
       : __ptr_(__u.release()),
-        __deleter_(std::forward<_Ep>(__u.get_deleter())) {}
+        __deleter_(std::forward<_Ep>(__u.get_deleter())),
+        __checker_(std::move(__u.__checker_)) {}
 
   template <class _Up,
             class _Ep,
@@ -434,6 +544,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 unique_ptr& operator=(unique_ptr<_Up, _Ep>&& __u) _NOEXCEPT {
     reset(__u.release());
     __deleter_ = std::forward<_Ep>(__u.get_deleter());
+    __checker_ = std::move(__u.__checker_);
     return *this;
   }
 
@@ -451,6 +562,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 __add_lvalue_reference_t<_Tp> operator[](size_t __i) const {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__checker_.__in_bounds(std::__to_address(__ptr_), __i),
+                                        "unique_ptr<T[]>::operator[](index): index out of range");
     return __ptr_[__i];
   }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer get() const _NOEXCEPT { return __ptr_; }
@@ -467,6 +580,8 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 pointer release() _NOEXCEPT {
     pointer __t = __ptr_;
     __ptr_      = pointer();
+    // The deleter and the optional bounds-checker are left unchanged. The bounds-checker
+    // will be reinitialized appropriately when/if the unique_ptr gets assigned-to or reset.
     return __t;
   }
 
@@ -474,6 +589,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void reset(_Pp __p) _NOEXCEPT {
     pointer __tmp = __ptr_;
     __ptr_        = __p;
+    __checker_    = _BoundsChecker();
     if (__tmp)
       __deleter_(__tmp);
   }
@@ -481,6 +597,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 void reset(nullptr_t = nullptr) _NOEXCEPT {
     pointer __tmp = __ptr_;
     __ptr_        = nullptr;
+    __checker_    = _BoundsChecker();
     if (__tmp)
       __deleter_(__tmp);
   }
@@ -489,6 +606,7 @@ class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp>
     using std::swap;
     swap(__ptr_, __u.__ptr_);
     swap(__deleter_, __u.__deleter_);
+    swap(__checker_, __u.__checker_);
   }
 };
 
@@ -645,7 +763,7 @@ template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_array_unknown_bound
 make_unique(size_t __n) {
   typedef __remove_extent_t<_Tp> _Up;
-  return unique_ptr<_Tp>(new _Up[__n]());
+  return unique_ptr<_Tp>(__private_constructor_tag(), new _Up[__n](), __n);
 }
 
 template <class _Tp, class... _Args>
@@ -664,7 +782,7 @@ make_unique_for_overwrite() {
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX23 typename __unique_if<_Tp>::__unique_array_unknown_bound
 make_unique_for_overwrite(size_t __n) {
-  return unique_ptr<_Tp>(new __remove_extent_t<_Tp>[__n]);
+  return unique_ptr<_Tp>(__private_constructor_tag(), new __remove_extent_t<_Tp>[__n], __n);
 }
 
 template <class _Tp, class... _Args>
diff --git a/libcxx/include/__std_clang_module b/libcxx/include/__std_clang_module
deleted file mode 100644
index a21ed26addfe8..0000000000000
--- a/libcxx/include/__std_clang_module
+++ /dev/null
@@ -1,193 +0,0 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// WARNING, this entire header is generated by
-// utils/generate_std_clang_module_header.py
-// DO NOT MODIFY!
-
-// This header should not be directly included, it's exclusively to import all
-// of the libc++ public clang modules for the `std` clang module to export. In
-// other words, it's to facilitate `@import std;` in Objective-C++ and `import std`
-// in Swift to expose all of the libc++ interfaces. This is generally not
-// recommended, however there are some clients that need to import all of libc++
-// without knowing what "all" is.
-#if !__building_module(std)
-#  error "Do not include this header directly, include individual headers instead"
-#endif
-
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-#include <algorithm>
-#include <any>
-#include <array>
-#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
-#  include <atomic>
-#endif
-#include <barrier>
-#include <bit>
-#include <bitset>
-#include <cassert>
-#include <ccomplex>
-#include <cctype>
-#include <cerrno>
-#include <cfenv>
-#include <cfloat>
-#include <charconv>
-#include <chrono>
-#include <cinttypes>
-#include <ciso646>
-#include <climits>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <clocale>
-#endif
-#include <cmath>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <codecvt>
-#endif
-#include <compare>
-#include <complex.h>
-#include <complex>
-#include <concepts>
-#include <condition_variable>
-#include <coroutine>
-#include <csetjmp>
-#include <csignal>
-#include <cstdarg>
-#include <cstdbool>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <ctgmath>
-#include <ctime>
-#include <ctype.h>
-#include <cuchar>
-#include <cwchar>
-#include <cwctype>
-#include <deque>
-#include <errno.h>
-#include <exception>
-#include <execution>
-#include <expected>
-#include <experimental/iterator>
-#include <experimental/memory>
-#include <experimental/propagate_const>
-#include <experimental/simd>
-#include <experimental/type_traits>
-#include <experimental/utility>
-#include <fenv.h>
-#include <filesystem>
-#include <float.h>
-#include <format>
-#include <forward_list>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <fstream>
-#endif
-#include <functional>
-#include <future>
-#include <initializer_list>
-#include <inttypes.h>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <iomanip>
-#endif
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <ios>
-#endif
-#include <iosfwd>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <iostream>
-#endif
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <istream>
-#endif
-#include <iterator>
-#include <latch>
-#include <limits>
-#include <list>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <locale.h>
-#endif
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <locale>
-#endif
-#include <map>
-#include <math.h>
-#include <mdspan>
-#include <memory>
-#include <memory_resource>
-#include <mutex>
-#include <new>
-#include <numbers>
-#include <numeric>
-#include <optional>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <ostream>
-#endif
-#include <print>
-#include <queue>
-#include <random>
-#include <ranges>
-#include <ratio>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <regex>
-#endif
-#include <scoped_allocator>
-#include <semaphore>
-#include <set>
-#include <shared_mutex>
-#include <source_location>
-#include <span>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <sstream>
-#endif
-#include <stack>
-#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
-#  include <stdatomic.h>
-#endif
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdexcept>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stop_token>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <streambuf>
-#endif
-#include <string.h>
-#include <string>
-#include <string_view>
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <strstream>
-#endif
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <syncstream>
-#endif
-#include <system_error>
-#include <tgmath.h>
-#include <thread>
-#include <tuple>
-#include <type_traits>
-#include <typeindex>
-#include <typeinfo>
-#include <uchar.h>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <valarray>
-#include <variant>
-#include <vector>
-#include <version>
-#include <wchar.h>
-#include <wctype.h>
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index 36fd035b7e51b..17d63ce0cf1c0 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -2020,10 +2020,10 @@ template <class BidirectionalIterator, class Compare>
 #endif
 
 #if _LIBCPP_STD_VER >= 23
-#  include <__algorithm/fold.h>
 #  include <__algorithm/ranges_contains_subrange.h>
 #  include <__algorithm/ranges_ends_with.h>
 #  include <__algorithm/ranges_find_last.h>
+#  include <__algorithm/ranges_fold.h>
 #  include <__algorithm/ranges_starts_with.h>
 #endif // _LIBCPP_STD_VER >= 23
 
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index ef4a242cf8bf7..aa05bde939f6c 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1,2124 +1,2235 @@
-// Main C++ standard library interfaces
-module std_algorithm [system] {
-  header "algorithm"
-  export *
-}
-module std_any [system] {
-  header "any"
-  export *
-}
-module std_array [system] {
-  header "array"
-  export *
-}
-module std_atomic [system] {
-  header "atomic"
-  export *
-}
-module std_barrier [system] {
-  header "barrier"
-  export *
-}
-module std_bit [system] {
-  header "bit"
-  export *
-}
-module std_bitset [system] {
-  header "bitset"
-  export *
-}
-module std_charconv [system] {
-  header "charconv"
-  module chars_format            { header "__charconv/chars_format.h" }
-  module from_chars_integral     { header "__charconv/from_chars_integral.h" }
-  module from_chars_result       { header "__charconv/from_chars_result.h" }
-  module tables                  { header "__charconv/tables.h" }
-  module to_chars                { header "__charconv/to_chars.h" }
-  module to_chars_base_10        { header "__charconv/to_chars_base_10.h" }
-  module to_chars_floating_point { header "__charconv/to_chars_floating_point.h" }
-  module to_chars_integral       { header "__charconv/to_chars_integral.h" }
-  module to_chars_result         { header "__charconv/to_chars_result.h" }
-  module traits                  { header "__charconv/traits.h" }
-  export *
-}
-module std_chrono [system] {
-  header "chrono"
-  export *
-}
-module std_codecvt [system] {
-  header "codecvt"
-  export *
-}
-module std_compare [system] {
-  header "compare"
-  export *
-}
-module std_complex [system] {
-  header "complex"
-  export *
-}
-module std_concepts [system] {
-  header "concepts"
-  export *
-}
-module std_condition_variable [system] {
-  header "condition_variable"
-  module condition_variable { header "__condition_variable/condition_variable.h" }
-  export *
-}
-module std_coroutine [system] {
-  header "coroutine"
-  module coroutine_handle      { header "__coroutine/coroutine_handle.h" }
-  module coroutine_traits      { header "__coroutine/coroutine_traits.h" }
-  module noop_coroutine_handle { header "__coroutine/noop_coroutine_handle.h" }
-  module trivial_awaitables    { header "__coroutine/trivial_awaitables.h" }
-  export *
-}
-module std_deque [system] {
-  header "deque"
-  export *
-}
-module std_exception [system] {
-  header "exception"
-  export *
-}
-module std_execution [system] {
-  header "execution"
-  export *
-}
-module std_expected [system] {
-  header "expected"
-  export *
-}
-module std_filesystem [system] {
-  header "filesystem"
-  module copy_options                 { header "__filesystem/copy_options.h" }
-  module directory_entry              { header "__filesystem/directory_entry.h" }
-  module directory_iterator           { header "__filesystem/directory_iterator.h" }
-  module directory_options            { header "__filesystem/directory_options.h" }
-  module file_status                  { header "__filesystem/file_status.h" }
-  module file_time_type               { header "__filesystem/file_time_type.h" }
-  module file_type                    { header "__filesystem/file_type.h" }
-  module filesystem_error             {
-    header "__filesystem/filesystem_error.h"
-    export std_private_memory_shared_ptr
-  }
-  module operations                   { header "__filesystem/operations.h" }
-  module path                         {
-    header "__filesystem/path.h"
-    export std_string // returned by various methods
-  }
-  module path_iterator                { header "__filesystem/path_iterator.h" }
-  module perm_options                 { header "__filesystem/perm_options.h" }
-  module perms                        { header "__filesystem/perms.h" }
-  module recursive_directory_iterator { header "__filesystem/recursive_directory_iterator.h" }
-  module space_info                   { header "__filesystem/space_info.h" }
-  module u8path                       { header "__filesystem/u8path.h" }
-  export *
-}
-module std_format [system] {
-  header "format"
-  export *
-}
-module std_forward_list [system] {
-  header "forward_list"
-  export *
-}
-module std_fstream [system] {
-  header "fstream"
-  export *
-}
-module std_functional [system] {
-  header "functional"
-  export *
-}
-module std_future [system] {
-  header "future"
-  export *
-}
-module std_initializer_list [system] {
-  header "initializer_list"
-  export *
-}
-module std_iomanip [system] {
-  header "iomanip"
-  export *
-}
-module std_ios [system] {
-  header "ios"
-  export *
-}
-module std_iosfwd [system] {
-  header "iosfwd"
-  export *
-}
-module std_iostream [system] {
-  header "iostream"
-  export *
-}
-module std_istream [system] {
-  header "istream"
-  export *
-}
-module std_iterator [system] {
-  header "iterator"
-  export *
-}
-module std_latch [system] {
-  header "latch"
-  export *
-}
-module std_limits [system] {
-  header "limits"
-  export *
-}
-module std_list [system] {
-  header "list"
-  export *
-}
-module std_locale [system] {
-  header "locale"
-  export *
-}
-module std_map [system] {
-  header "map"
-  export *
+// This module contains headers related to the configuration of the library. These headers
+// are free of any dependency on the rest of libc++.
+module std_config [system] {
+  textual header "__config"
+  textual header "__configuration/abi.h"
+  textual header "__configuration/availability.h"
+  textual header "__configuration/compiler.h"
+  textual header "__configuration/language.h"
+  textual header "__configuration/platform.h"
+  textual header "version"
 }
-module std_mdspan [system] {
-  header "mdspan"
-  module default_accessor { header "__mdspan/default_accessor.h" }
-  module extents          { header "__mdspan/extents.h" }
-  module fwd              { header "__fwd/mdspan.h" }
-  module layout_left      { header "__mdspan/layout_left.h" }
-  module layout_right     { header "__mdspan/layout_right.h" }
-  module layout_stride    { header "__mdspan/layout_stride.h" }
-  module mdspan           {
-    header "__mdspan/mdspan.h"
-    export std_array // for strides()
+
+module std_core [system] {
+  module cstddef {
+    module byte         { header "__cstddef/byte.h" }
+    module max_align_t  { header "__cstddef/max_align_t.h" }
+    module nullptr_t    { header "__cstddef/nullptr_t.h" }
+    module ptrdiff_t    { header "__cstddef/ptrdiff_t.h" }
+    module size_t       { header "__cstddef/size_t.h" }
   }
-  export *
-}
-module std_memory [system] {
-  header "memory"
-  export *
-}
-module std_memory_resource [system] {
-  header "memory_resource"
-  export *
-}
-module std_mutex [system] {
-  header "mutex"
-  export *
-}
-module std_new [system] {
-  header "new"
-  export *
-}
-module std_numbers [system] {
-  header "numbers"
-  export *
-}
-module std_numeric [system] {
-  header "numeric"
-  export *
-}
-module std_optional [system] {
-  header "optional"
-  export *
-}
-module std_ostream [system] {
-  header "ostream"
-  export *
-}
-module std_print [system] {
-  header "print"
-  export *
-}
-module std_queue [system] {
-  header "queue"
-  export *
-}
-module std_random [system] {
-  header "random"
-  export *
-}
-module std_ranges [system] {
-  header "ranges"
-  export *
-}
-module std_ratio [system] {
-  header "ratio"
-  export *
-}
-module std_regex [system] {
-  header "regex"
-  export *
-}
-module std_scoped_allocator [system] {
-  header "scoped_allocator"
-  export *
-}
-module std_semaphore [system] {
-  header "semaphore"
-  export *
-}
-module std_set [system] {
-  header "set"
-  export *
-}
-module std_shared_mutex [system] {
-  header "shared_mutex"
-  export std_version
-}
-module std_source_location [system] {
-  header "source_location"
-  export *
-}
-module std_span [system] {
-  header "span"
-  export std_private_ranges_enable_borrowed_range
-  export std_version
-  export std_private_span_span_fwd
-}
-module std_sstream [system] {
-  header "sstream"
-  export *
-}
-module std_stack [system] {
-  header "stack"
-  export *
-}
-module std_stdexcept [system] {
-  header "stdexcept"
-  export *
-}
-module std_stop_token [system] {
-  header "stop_token"
-  private header "__stop_token/atomic_unique_lock.h"
-  private header "__stop_token/intrusive_list_view.h"
-  private header "__stop_token/intrusive_shared_ptr.h"
-  private header "__stop_token/stop_callback.h"
-  private header "__stop_token/stop_source.h"
-  private header "__stop_token/stop_state.h"
-  private header "__stop_token/stop_token.h"
-  export *
-}
-module std_streambuf [system] {
-  header "streambuf"
-  export *
-}
-module std_string [system] {
-  header "string"
-  export *
-}
-module std_string_view [system] {
-  header "string_view"
-  export *
-}
-module std_strstream [system] {
-  header "strstream"
-  export *
-}
-module std_syncstream [system] {
-  header "syncstream"
-  export *
-}
-module std_system_error [system] {
-  header "system_error"
-  export *
-}
-module std_thread [system] {
-  header "thread"
-  export *
-}
-module std_tuple [system] {
-  header "tuple"
-  export *
-}
-module std_type_traits [system] {
-  header "type_traits"
-  export *
-}
-module std_typeindex [system] {
-  header "typeindex"
-  export *
-}
-module std_typeinfo [system] {
-  header "typeinfo"
-  export *
-}
-module std_unordered_map [system] {
-  header "unordered_map"
-  export *
-}
-module std_unordered_set [system] {
-  header "unordered_set"
-  export *
-}
-module std_utility [system] {
-  header "utility"
-  export *
-}
-module std_valarray [system] {
-  header "valarray"
-  export *
-}
-module std_variant [system] {
-  header "variant"
-  export *
-}
-module std_vector [system] {
-  header "vector"
-  export *
-}
-module std_version [system] {
-  header "version"
-  export *
-}
 
-// C standard library interface wrappers
-module std_cassert [system] {
-  // <cassert>'s use of NDEBUG requires textual inclusion.
-  textual header "cassert"
-}
-module std_ccomplex [system] {
-  header "ccomplex"
-  export *
-}
-module std_cctype [system] {
-  header "cctype"
-  export *
-}
-module std_cerrno [system] {
-  header "cerrno"
-  export *
-}
-module std_cfenv [system] {
-  header "cfenv"
-  export *
-}
-module std_cfloat [system] {
-  header "cfloat"
-  export *
-}
-module std_cinttypes [system] {
-  header "cinttypes"
-  export *
-}
-module std_ciso646 [system] {
-  header "ciso646"
-  export *
-}
-module std_climits [system] {
-  header "climits"
-  export *
-}
-module std_clocale [system] {
-  header "clocale"
-  export *
-}
-module std_cmath [system] {
-  header "cmath"
-  export *
-}
-module std_csetjmp [system] {
-  header "csetjmp"
-  export *
-}
-module std_csignal [system] {
-  header "csignal"
-  export *
-}
-// FIXME: <cstdalign> is missing.
-module std_cstdarg [system] {
-  header "cstdarg"
-  export *
-}
-module std_cstdbool [system] {
-  header "cstdbool"
-  export *
-}
-module std_cstddef [system] {
-  header "cstddef"
-  module byte         { header "__cstddef/byte.h" }
-  module max_align_t  { header "__cstddef/max_align_t.h" }
-  module nullptr_t    { header "__cstddef/nullptr_t.h" }
-  module ptrdiff_t    { header "__cstddef/ptrdiff_t.h" }
-  module size_t       { header "__cstddef/size_t.h" }
-  export *
-}
-module std_cstdint [system] {
-  header "cstdint"
-  export *
-}
-module std_cstdio [system] {
-  header "cstdio"
-  export *
-}
-module std_cstdlib [system] {
-  header "cstdlib"
-  export *
-}
-module std_cstring [system] {
-  header "cstring"
-  export *
-}
-module std_ctgmath [system] {
-  header "ctgmath"
-  export *
-}
-module std_ctime [system] {
-  header "ctime"
-  export *
-}
-module std_cuchar [system] {
-  header "cuchar"
-  export *
-}
-module std_cwchar [system] {
-  header "cwchar"
-  export *
-}
-module std_cwctype [system] {
-  header "cwctype"
-  export *
-}
+  module cstdint {
+    header "cstdint"
+    export *
+  }
 
-// C standard library interfaces augmented/replaced in C++
-// <assert.h> provided by C library.
-module std_complex_h [system] {
-  header "complex.h"
-  export *
-}
-module std_ctype_h [system] {
-  header "ctype.h"
-  export *
-}
-module std_errno_h [system] {
-  header "errno.h"
-  export *
-}
-module std_fenv_h [system] {
-  header "fenv.h"
-  export *
-}
-module std_float_h [system] {
-  header "float.h"
-  export *
-}
-module std_inttypes_h [system] {
-  header "inttypes.h"
-  export *
-}
-// <iso646.h> provided by compiler.
-module std_locale_h [system] {
-  header "locale.h"
-  export *
-}
-module std_math_h [system] {
-  header "math.h"
-  export *
-}
-// <setjmp.h> provided by C library.
-// <signal.h> provided by C library.
-// FIXME: <stdalign.h> is missing.
-// <stdarg.h> provided by compiler.
-module std_stdatomic_h [system] {
-  header "stdatomic.h"
-  export *
-}
-module std_stdbool_h [system] {
-  // <stdbool.h>'s __bool_true_false_are_defined macro requires textual inclusion.
-  textual header "stdbool.h"
-  export *
-}
-module std_stddef_h [system] {
-  // <stddef.h>'s __need_* macros require textual inclusion.
-  textual header "stddef.h"
-  export *
-}
-module std_stdint_h [system] {
-  header "stdint.h"
-  export *
-}
-module std_stdio_h [system] {
-  // <stdio.h>'s __need_* macros require textual inclusion.
-  textual header "stdio.h"
-  export *
-}
-module std_stdlib_h [system] {
-  // <stdlib.h>'s __need_* macros require textual inclusion.
-  textual header "stdlib.h"
-  export *
-}
-module std_string_h [system] {
-  header "string.h"
-  export *
-}
-module std_tgmath_h [system] {
-  header "tgmath.h"
-  export *
-}
-module std_uchar_h [system] {
-  header "uchar.h"
-  export *
-}
-// <time.h> provided by C library.
-module std_wchar_h [system] {
-  // <wchar.h>'s __need_* macros require textual inclusion.
-  textual header "wchar.h"
-  export *
-}
-module std_wctype_h [system] {
-  header "wctype.h"
-  export *
-}
+  module fwd {
+    module byte         { header "__fwd/byte.h" }
+    module functional   { header "__fwd/functional.h" }
+    module pair         { header "__fwd/pair.h" }
+    module tuple        { header "__fwd/tuple.h" }
+  }
+
+  module limits {
+    header "limits"
+    export *
+  }
+
+  module math {
+    module abs                              { header "__math/abs.h" }
+    module copysign                         { header "__math/copysign.h" }
+    module error_functions                  { header "__math/error_functions.h" }
+    module exponential_functions            { header "__math/exponential_functions.h" }
+    module fdim                             { header "__math/fdim.h" }
+    module fma                              { header "__math/fma.h" }
+    module gamma                            { header "__math/gamma.h" }
+    module hyperbolic_functions             { header "__math/hyperbolic_functions.h" }
+    module hypot                            { header "__math/hypot.h" }
+    module inverse_hyperbolic_functions     { header "__math/inverse_hyperbolic_functions.h" }
+    module inverse_trigonometric_functions  { header "__math/inverse_trigonometric_functions.h" }
+    module logarithms                       { header "__math/logarithms.h" }
+    module min_max                          { header "__math/min_max.h" }
+    module modulo                           { header "__math/modulo.h" }
+    module remainder                        { header "__math/remainder.h" }
+    module roots                            { header "__math/roots.h" }
+    module rounding_functions               { header "__math/rounding_functions.h" }
+    module special_functions                { header "__math/special_functions.h" }
+    module traits                           { header "__math/traits.h" }
+    module trigonometric_functions          { header "__math/trigonometric_functions.h" }
+  }
+
+  module type_traits {
+    module add_const                                  { header "__type_traits/add_const.h" }
+    module add_cv                                     { header "__type_traits/add_cv.h" }
+    module add_lvalue_reference                       { header "__type_traits/add_lvalue_reference.h" }
+    module add_pointer                                { header "__type_traits/add_pointer.h" }
+    module add_rvalue_reference                       { header "__type_traits/add_rvalue_reference.h" }
+    module add_volatile                               { header "__type_traits/add_volatile.h" }
+    module aligned_storage                            { header "__type_traits/aligned_storage.h" }
+    module aligned_union                              { header "__type_traits/aligned_union.h" }
+    module alignment_of                               { header "__type_traits/alignment_of.h" }
+    module can_extract_key                            { header "__type_traits/can_extract_key.h" }
+    module common_reference                           { header "__type_traits/common_reference.h" }
+    module common_type                                { header "__type_traits/common_type.h" }
+    module conditional                                { header "__type_traits/conditional.h" }
+    module conjunction                                { header "__type_traits/conjunction.h" }
+    module copy_cv                                    { header "__type_traits/copy_cv.h" }
+    module copy_cvref                                 { header "__type_traits/copy_cvref.h" }
+    module datasizeof                                 { header "__type_traits/datasizeof.h" }
+    module decay                                      { header "__type_traits/decay.h" }
+    module dependent_type                             { header "__type_traits/dependent_type.h" }
+    module desugars_to                                { header "__type_traits/desugars_to.h" }
+    module disjunction                                { header "__type_traits/disjunction.h" }
+    module enable_if                                  { header "__type_traits/enable_if.h" }
+    module extent                                     { header "__type_traits/extent.h" }
+    module has_unique_object_representation           { header "__type_traits/has_unique_object_representation.h" }
+    module has_virtual_destructor                     { header "__type_traits/has_virtual_destructor.h" }
+    module integral_constant                          { header "__type_traits/integral_constant.h" }
+    module invoke                                     { header "__type_traits/invoke.h" }
+    module is_abstract {
+      header "__type_traits/is_abstract.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_aggregate {
+      header "__type_traits/is_aggregate.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_allocator {
+      header "__type_traits/is_allocator.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_always_bitcastable {
+      header "__type_traits/is_always_bitcastable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_arithmetic {
+      header "__type_traits/is_arithmetic.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_array {
+      header "__type_traits/is_array.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_assignable {
+      header "__type_traits/is_assignable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_base_of {
+      header "__type_traits/is_base_of.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_bounded_array {
+      header "__type_traits/is_bounded_array.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_callable {
+      header "__type_traits/is_callable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_char_like_type {
+      header "__type_traits/is_char_like_type.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_class {
+      header "__type_traits/is_class.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_compound {
+      header "__type_traits/is_compound.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_const {
+      header "__type_traits/is_const.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_constant_evaluated {
+      header "__type_traits/is_constant_evaluated.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_constructible {
+      header "__type_traits/is_constructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_convertible {
+      header "__type_traits/is_convertible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_core_convertible {
+      header "__type_traits/is_core_convertible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_destructible {
+      header "__type_traits/is_destructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_empty {
+      header "__type_traits/is_empty.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_enum {
+      header "__type_traits/is_enum.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_equality_comparable {
+      header "__type_traits/is_equality_comparable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_execution_policy {
+      header "__type_traits/is_execution_policy.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_final {
+      header "__type_traits/is_final.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_floating_point {
+      header "__type_traits/is_floating_point.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_function {
+      header "__type_traits/is_function.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_fundamental {
+      header "__type_traits/is_fundamental.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_implicitly_default_constructible {
+      header "__type_traits/is_implicitly_default_constructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_integral {
+      header "__type_traits/is_integral.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_literal_type {
+      header "__type_traits/is_literal_type.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_member_pointer {
+      header "__type_traits/is_member_pointer.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_nothrow_assignable {
+      header "__type_traits/is_nothrow_assignable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_nothrow_constructible {
+      header "__type_traits/is_nothrow_constructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_nothrow_convertible {
+      header "__type_traits/is_nothrow_convertible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_nothrow_destructible {
+      header "__type_traits/is_nothrow_destructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_null_pointer {
+      header "__type_traits/is_null_pointer.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_object {
+      header "__type_traits/is_object.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_pod {
+      header "__type_traits/is_pod.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_pointer {
+      header "__type_traits/is_pointer.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_polymorphic {
+      header "__type_traits/is_polymorphic.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_primary_template {
+      header "__type_traits/is_primary_template.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_reference_wrapper {
+      header "__type_traits/is_reference_wrapper.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_reference {
+      header "__type_traits/is_reference.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_referenceable {
+      header "__type_traits/is_referenceable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_same {
+      header "__type_traits/is_same.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_scalar {
+      header "__type_traits/is_scalar.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_signed_integer {
+      header "__type_traits/is_signed_integer.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_signed {
+      header "__type_traits/is_signed.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_specialization {
+      header "__type_traits/is_specialization.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_standard_layout {
+      header "__type_traits/is_standard_layout.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_swappable {
+      header "__type_traits/is_swappable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivial {
+      header "__type_traits/is_trivial.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_assignable {
+      header "__type_traits/is_trivially_assignable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_constructible {
+      header "__type_traits/is_trivially_constructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_copyable {
+      header "__type_traits/is_trivially_copyable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_destructible {
+      header "__type_traits/is_trivially_destructible.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_lexicographically_comparable {
+      header "__type_traits/is_trivially_lexicographically_comparable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_trivially_relocatable {
+      header "__type_traits/is_trivially_relocatable.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_unbounded_array {
+      header "__type_traits/is_unbounded_array.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_union {
+      header "__type_traits/is_union.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_unsigned_integer {
+      header "__type_traits/is_unsigned_integer.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_unsigned {
+      header "__type_traits/is_unsigned.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_valid_expansion {
+      header "__type_traits/is_valid_expansion.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_void {
+      header "__type_traits/is_void.h"
+      export std_core.type_traits.integral_constant
+    }
+    module is_volatile {
+      header "__type_traits/is_volatile.h"
+      export std_core.type_traits.integral_constant
+    }
+    module lazy                                       { header "__type_traits/lazy.h" }
+    module make_32_64_or_128_bit                      { header "__type_traits/make_32_64_or_128_bit.h" }
+    module make_const_lvalue_ref                      { header "__type_traits/make_const_lvalue_ref.h" }
+    module make_signed                                { header "__type_traits/make_signed.h" }
+    module make_unsigned                              { header "__type_traits/make_unsigned.h" }
+    module maybe_const                                { header "__type_traits/maybe_const.h" }
+    module nat                                        { header "__type_traits/nat.h" }
+    module negation                                   { header "__type_traits/negation.h" }
+    module promote                                    { header "__type_traits/promote.h" }
+    module rank                                       { header "__type_traits/rank.h" }
+    module remove_all_extents                         { header "__type_traits/remove_all_extents.h" }
+    module remove_const_ref                           { header "__type_traits/remove_const_ref.h" }
+    module remove_const                               { header "__type_traits/remove_const.h" }
+    module remove_cv                                  { header "__type_traits/remove_cv.h" }
+    module remove_cvref                               { header "__type_traits/remove_cvref.h" }
+    module remove_extent                              { header "__type_traits/remove_extent.h" }
+    module remove_pointer                             { header "__type_traits/remove_pointer.h" }
+    module remove_reference                           { header "__type_traits/remove_reference.h" }
+    module remove_volatile                            { header "__type_traits/remove_volatile.h" }
+    module result_of                                  { header "__type_traits/result_of.h" }
+    module strip_signature                            { header "__type_traits/strip_signature.h" }
+    module type_identity                              { header "__type_traits/type_identity.h" }
+    module type_list                                  { header "__type_traits/type_list.h" }
+    module underlying_type                            { header "__type_traits/underlying_type.h" }
+    module unwrap_ref                                 { header "__type_traits/unwrap_ref.h" }
+    module void_t                                     { header "__type_traits/void_t.h" }
+
+    header "type_traits"
+    export *
+  } // module type_traits
+
+  // Only the truly dependency-free parts of __utility are here
+  module utility_core {
+    module declval  { header "__utility/declval.h" }
+    module empty    { header "__utility/empty.h" }
+    module forward  { header "__utility/forward.h" }
+  }
+} // module std_core
+
+module std [system] {
+  module algorithm {
+    module adjacent_find                          { header "__algorithm/adjacent_find.h" }
+    module all_of                                 { header "__algorithm/all_of.h" }
+    module any_of                                 { header "__algorithm/any_of.h" }
+    module binary_search                          { header "__algorithm/binary_search.h" }
+    module clamp                                  { header "__algorithm/clamp.h" }
+    module comp_ref_type                          { header "__algorithm/comp_ref_type.h" }
+    module comp                                   { header "__algorithm/comp.h" }
+    module copy_backward                          { header "__algorithm/copy_backward.h" }
+    module copy_if                                { header "__algorithm/copy_if.h" }
+    module copy_move_common                       { header "__algorithm/copy_move_common.h" }
+    module copy_n                                 { header "__algorithm/copy_n.h" }
+    module copy                                   { header "__algorithm/copy.h" }
+    module count_if                               { header "__algorithm/count_if.h" }
+    module count                                  { header "__algorithm/count.h" }
+    module equal_range                            { header "__algorithm/equal_range.h" }
+    module equal                                  { header "__algorithm/equal.h" }
+    module fill_n                                 { header "__algorithm/fill_n.h" }
+    module fill                                   { header "__algorithm/fill.h" }
+    module find_end                               { header "__algorithm/find_end.h" }
+    module find_first_of                          { header "__algorithm/find_first_of.h" }
+    module find_if_not                            { header "__algorithm/find_if_not.h" }
+    module find_if                                { header "__algorithm/find_if.h" }
+    module find_segment_if                        { header "__algorithm/find_segment_if.h" }
+    module find                                   { header "__algorithm/find.h" }
+    module for_each_n                             { header "__algorithm/for_each_n.h" }
+    module for_each_segment                       { header "__algorithm/for_each_segment.h" }
+    module for_each                               { header "__algorithm/for_each.h" }
+    module generate_n                             { header "__algorithm/generate_n.h" }
+    module generate                               { header "__algorithm/generate.h" }
+    module half_positive                          { header "__algorithm/half_positive.h" }
+    module in_found_result                        { header "__algorithm/in_found_result.h" }
+    module in_fun_result                          { header "__algorithm/in_fun_result.h" }
+    module in_in_out_result                       { header "__algorithm/in_in_out_result.h" }
+    module in_in_result                           { header "__algorithm/in_in_result.h" }
+    module in_out_out_result                      { header "__algorithm/in_out_out_result.h" }
+    module in_out_result                          { header "__algorithm/in_out_result.h" }
+    module includes                               { header "__algorithm/includes.h" }
+    module inplace_merge                          { header "__algorithm/inplace_merge.h" }
+    module is_heap_until                          { header "__algorithm/is_heap_until.h" }
+    module is_heap                                { header "__algorithm/is_heap.h" }
+    module is_partitioned                         { header "__algorithm/is_partitioned.h" }
+    module is_permutation                         { header "__algorithm/is_permutation.h" }
+    module is_sorted_until                        { header "__algorithm/is_sorted_until.h" }
+    module is_sorted                              { header "__algorithm/is_sorted.h" }
+    module iter_swap                              { header "__algorithm/iter_swap.h" }
+    module iterator_operations {
+      header "__algorithm/iterator_operations.h"
+      export std.iterator.advance
+      export std.iterator.distance
+      export std.iterator.iter_move
+      export std.iterator.iter_swap
+      export std.iterator.next
+      export std.iterator.prev
+    }
+    module lexicographical_compare_three_way      { header "__algorithm/lexicographical_compare_three_way.h" }
+    module lexicographical_compare                { header "__algorithm/lexicographical_compare.h" }
+    module lower_bound                            { header "__algorithm/lower_bound.h" }
+    module make_heap                              { header "__algorithm/make_heap.h" }
+    module make_projected                         { header "__algorithm/make_projected.h" }
+    module max_element                            { header "__algorithm/max_element.h" }
+    module max                                    { header "__algorithm/max.h" }
+    module merge                                  { header "__algorithm/merge.h" }
+    module min_element                            { header "__algorithm/min_element.h" }
+    module min_max_result                         { header "__algorithm/min_max_result.h" }
+    module min                                    { header "__algorithm/min.h" }
+    module minmax_element                         { header "__algorithm/minmax_element.h" }
+    module minmax {
+      header "__algorithm/minmax.h"
+      export std.utility.pair // return type
+    }
+    module mismatch {
+      header "__algorithm/mismatch.h"
+      export std.utility.pair // return type
+    }
+    module move_backward                          { header "__algorithm/move_backward.h" }
+    module move                                   { header "__algorithm/move.h" }
+    module next_permutation                       { header "__algorithm/next_permutation.h" }
+    module none_of                                { header "__algorithm/none_of.h" }
+    module nth_element                            { header "__algorithm/nth_element.h" }
+    module partial_sort_copy                      { header "__algorithm/partial_sort_copy.h" }
+    module partial_sort                           { header "__algorithm/partial_sort.h" }
+    module partition_copy                         { header "__algorithm/partition_copy.h" }
+    module partition_point                        { header "__algorithm/partition_point.h" }
+    module partition                              { header "__algorithm/partition.h" }
+    module pop_heap                               { header "__algorithm/pop_heap.h" }
+    module prev_permutation                       { header "__algorithm/prev_permutation.h" }
+    module pstl                                   { header "__algorithm/pstl.h" }
+    module push_heap                              { header "__algorithm/push_heap.h" }
+    module ranges_adjacent_find                   { header "__algorithm/ranges_adjacent_find.h" }
+    module ranges_all_of                          { header "__algorithm/ranges_all_of.h" }
+    module ranges_any_of                          { header "__algorithm/ranges_any_of.h" }
+    module ranges_binary_search {
+      header "__algorithm/ranges_binary_search.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_clamp {
+      header "__algorithm/ranges_clamp.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_contains_subrange {
+      header "__algorithm/ranges_contains_subrange.h"
+    }
+    module ranges_contains {
+      header "__algorithm/ranges_contains.h"
+    }
+    module ranges_copy_backward {
+      header "__algorithm/ranges_copy_backward.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_copy_if {
+      header "__algorithm/ranges_copy_if.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_copy_n {
+      header "__algorithm/ranges_copy_n.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_copy {
+      header "__algorithm/ranges_copy.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_count_if                        { header "__algorithm/ranges_count_if.h" }
+    module ranges_count                           { header "__algorithm/ranges_count.h" }
+    module ranges_ends_with                       { header "__algorithm/ranges_ends_with.h" }
+    module ranges_equal_range {
+      header "__algorithm/ranges_equal_range.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_equal {
+      header "__algorithm/ranges_equal.h"
+      export std.functional.identity
+    }
+    module ranges_fill_n                          { header "__algorithm/ranges_fill_n.h" }
+    module ranges_fill                            { header "__algorithm/ranges_fill.h" }
+    module ranges_find_end                        { header "__algorithm/ranges_find_end.h" }
+    module ranges_find_first_of                   { header "__algorithm/ranges_find_first_of.h" }
+    module ranges_find_if_not                     { header "__algorithm/ranges_find_if_not.h" }
+    module ranges_find_if                         { header "__algorithm/ranges_find_if.h" }
+    module ranges_find_last                       { header "__algorithm/ranges_find_last.h" }
+    module ranges_find                            { header "__algorithm/ranges_find.h" }
+    module ranges_fold                            { header "__algorithm/ranges_fold.h" }
+    module ranges_for_each_n {
+      header "__algorithm/ranges_for_each_n.h"
+      export std.algorithm.in_fun_result
+    }
+    module ranges_for_each {
+      header "__algorithm/ranges_for_each.h"
+      export std.algorithm.in_fun_result
+    }
+    module ranges_generate_n {
+      header "__algorithm/ranges_generate_n.h"
+    }
+    module ranges_generate {
+      header "__algorithm/ranges_generate.h"
+    }
+    module ranges_includes {
+      header "__algorithm/ranges_includes.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_inplace_merge {
+      header "__algorithm/ranges_inplace_merge.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_is_heap_until {
+      header "__algorithm/ranges_is_heap_until.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_is_heap {
+      header "__algorithm/ranges_is_heap.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_is_partitioned {
+      header "__algorithm/ranges_is_partitioned.h"
+    }
+    module ranges_is_permutation {
+      header "__algorithm/ranges_is_permutation.h"
+    }
+    module ranges_is_sorted_until {
+      header "__algorithm/ranges_is_sorted_until.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_is_sorted {
+      header "__algorithm/ranges_is_sorted.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_iterator_concept {
+      header "__algorithm/ranges_iterator_concept.h"
+    }
+    module ranges_lexicographical_compare {
+      header "__algorithm/ranges_lexicographical_compare.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_lower_bound {
+      header "__algorithm/ranges_lower_bound.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_make_heap {
+      header "__algorithm/ranges_make_heap.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_max_element {
+      header "__algorithm/ranges_max_element.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_max {
+      header "__algorithm/ranges_max.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_merge {
+      header "__algorithm/ranges_merge.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_in_out_result
+    }
+    module ranges_min_element {
+      header "__algorithm/ranges_min_element.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_min {
+      header "__algorithm/ranges_min.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_minmax_element {
+      header "__algorithm/ranges_minmax_element.h"
+      export std.functional.ranges_operations
+      export std.algorithm.min_max_result
+    }
+    module ranges_minmax {
+      header "__algorithm/ranges_minmax.h"
+      export std.functional.ranges_operations
+      export std.algorithm.min_max_result
+    }
+    module ranges_mismatch {
+      header "__algorithm/ranges_mismatch.h"
+      export std.algorithm.in_in_result
+    }
+    module ranges_move_backward {
+      header "__algorithm/ranges_move_backward.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_move {
+      header "__algorithm/ranges_move.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_next_permutation {
+      header "__algorithm/ranges_next_permutation.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_found_result
+    }
+    module ranges_none_of {
+      header "__algorithm/ranges_none_of.h"
+    }
+    module ranges_nth_element {
+      header "__algorithm/ranges_nth_element.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_partial_sort_copy {
+      header "__algorithm/ranges_partial_sort_copy.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_partial_sort {
+      header "__algorithm/ranges_partial_sort.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_partition_copy {
+      header "__algorithm/ranges_partition_copy.h"
+      export std.algorithm.in_out_out_result
+    }
+    module ranges_partition_point {
+      header "__algorithm/ranges_partition_point.h"
+    }
+    module ranges_partition {
+      header "__algorithm/ranges_partition.h"
+    }
+    module ranges_pop_heap {
+      header "__algorithm/ranges_pop_heap.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_prev_permutation {
+      header "__algorithm/ranges_prev_permutation.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_found_result
+    }
+    module ranges_push_heap {
+      header "__algorithm/ranges_push_heap.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_remove_copy_if {
+      header "__algorithm/ranges_remove_copy_if.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_remove_copy {
+      header "__algorithm/ranges_remove_copy.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_remove_if {
+      header "__algorithm/ranges_remove_if.h"
+    }
+    module ranges_remove {
+      header "__algorithm/ranges_remove.h"
+    }
+    module ranges_replace_copy_if {
+      header "__algorithm/ranges_replace_copy_if.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_replace_copy {
+      header "__algorithm/ranges_replace_copy.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_replace_if {
+      header "__algorithm/ranges_replace_if.h"
+    }
+    module ranges_replace {
+      header "__algorithm/ranges_replace.h"
+    }
+    module ranges_reverse_copy {
+      header "__algorithm/ranges_reverse_copy.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_reverse {
+      header "__algorithm/ranges_reverse.h"
+    }
+    module ranges_rotate_copy {
+      header "__algorithm/ranges_rotate_copy.h"
+      export std.algorithm.in_out_result
+    }
+    module ranges_rotate                          { header "__algorithm/ranges_rotate.h" }
+    module ranges_sample                          { header "__algorithm/ranges_sample.h" }
+    module ranges_search_n                        { header "__algorithm/ranges_search_n.h" }
+    module ranges_search                          { header "__algorithm/ranges_search.h" }
+    module ranges_set_difference {
+      header "__algorithm/ranges_set_difference.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_out_result
+    }
+    module ranges_set_intersection {
+      header "__algorithm/ranges_set_intersection.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_in_out_result
+    }
+    module ranges_set_symmetric_difference {
+      header "__algorithm/ranges_set_symmetric_difference.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_in_out_result
+    }
+    module ranges_set_union {
+      header "__algorithm/ranges_set_union.h"
+      export std.functional.ranges_operations
+      export std.algorithm.in_in_out_result
+    }
+    module ranges_shuffle {
+      header "__algorithm/ranges_shuffle.h"
+    }
+    module ranges_sort_heap {
+      header "__algorithm/ranges_sort_heap.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_sort {
+      header "__algorithm/ranges_sort.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_stable_partition {
+      header "__algorithm/ranges_stable_partition.h"
+    }
+    module ranges_stable_sort {
+      header "__algorithm/ranges_stable_sort.h"
+      export std.functional.ranges_operations
+    }
+    module ranges_starts_with {
+      header "__algorithm/ranges_starts_with.h"
+    }
+    module ranges_swap_ranges {
+      header "__algorithm/ranges_swap_ranges.h"
+      export std.algorithm.in_in_result
+    }
+    module ranges_transform {
+      header "__algorithm/ranges_transform.h"
+      export std.algorithm.in_out_result
+      export std.algorithm.in_in_out_result
+    }
+    module ranges_unique_copy {
+      header "__algorithm/ranges_unique_copy.h"
+    }
+    module ranges_unique {
+      header "__algorithm/ranges_unique.h"
+    }
+    module ranges_upper_bound {
+      header "__algorithm/ranges_upper_bound.h"
+      export std.functional.ranges_operations
+    }
+    module remove_copy_if                         { header "__algorithm/remove_copy_if.h" }
+    module remove_copy                            { header "__algorithm/remove_copy.h" }
+    module remove_if                              { header "__algorithm/remove_if.h" }
+    module remove                                 { header "__algorithm/remove.h" }
+    module replace_copy_if                        { header "__algorithm/replace_copy_if.h" }
+    module replace_copy                           { header "__algorithm/replace_copy.h" }
+    module replace_if                             { header "__algorithm/replace_if.h" }
+    module replace                                { header "__algorithm/replace.h" }
+    module reverse_copy                           { header "__algorithm/reverse_copy.h" }
+    module reverse                                { header "__algorithm/reverse.h" }
+    module rotate_copy                            { header "__algorithm/rotate_copy.h" }
+    module rotate                                 { header "__algorithm/rotate.h" }
+    module sample                                 { header "__algorithm/sample.h" }
+    module search_n                               { header "__algorithm/search_n.h" }
+    module search                                 { header "__algorithm/search.h" }
+    module set_difference                         { header "__algorithm/set_difference.h" }
+    module set_intersection                       { header "__algorithm/set_intersection.h" }
+    module set_symmetric_difference               { header "__algorithm/set_symmetric_difference.h" }
+    module set_union                              { header "__algorithm/set_union.h" }
+    module shift_left                             { header "__algorithm/shift_left.h" }
+    module shift_right                            { header "__algorithm/shift_right.h" }
+    module shuffle                                { header "__algorithm/shuffle.h" }
+    module sift_down                              { header "__algorithm/sift_down.h" }
+    module simd_utils                             { header "__algorithm/simd_utils.h" }
+    module sort_heap                              { header "__algorithm/sort_heap.h" }
+    module sort                                   { header "__algorithm/sort.h" }
+    module stable_partition                       { header "__algorithm/stable_partition.h" }
+    module stable_sort                            { header "__algorithm/stable_sort.h" }
+    module swap_ranges                            { header "__algorithm/swap_ranges.h" }
+    module three_way_comp_ref_type                { header "__algorithm/three_way_comp_ref_type.h" }
+    module transform                              { header "__algorithm/transform.h" }
+    module uniform_random_bit_generator_adaptor   { header "__algorithm/uniform_random_bit_generator_adaptor.h" }
+    module unique_copy                            { header "__algorithm/unique_copy.h" }
+    module unique                                 { header "__algorithm/unique.h" }
+    module unwrap_iter                            { header "__algorithm/unwrap_iter.h" }
+    module unwrap_range                           { header "__algorithm/unwrap_range.h" }
+    module upper_bound                            { header "__algorithm/upper_bound.h" }
+
+    header "algorithm"
+    export *
+  } // module algorithm
+
+  module any {
+    header "any"
+    export *
+  }
+
+  module array {
+    module fwd { header "__fwd/array.h" }
+
+    header "array"
+    export *
+  }
+
+  module atomic {
+    module aliases                { header "__atomic/aliases.h" }
+    module atomic_base            { header "__atomic/atomic_base.h" }
+    module atomic_flag            { header "__atomic/atomic_flag.h" }
+    module atomic_init            { header "__atomic/atomic_init.h" }
+    module atomic_lock_free       { header "__atomic/atomic_lock_free.h" }
+    module atomic_ref             { header "__atomic/atomic_ref.h" }
+    module atomic_sync            { header "__atomic/atomic_sync.h" }
+    module atomic {
+      header "__atomic/atomic.h"
+      export std.atomic.atomic_base // most of std::atomic methods are defined there
+    }
+    module check_memory_order     { header "__atomic/check_memory_order.h" }
+    module contention_t           { header "__atomic/contention_t.h" }
+    module cxx_atomic_impl        { header "__atomic/cxx_atomic_impl.h" }
+    module fence                  { header "__atomic/fence.h" }
+    module is_always_lock_free    { header "__atomic/is_always_lock_free.h" }
+    module kill_dependency        { header "__atomic/kill_dependency.h" }
+    module memory_order           { header "__atomic/memory_order.h" }
+    module to_gcc_order           { header "__atomic/to_gcc_order.h" }
+
+    header "atomic"
+    export *
+  }
+
+  module barrier {
+    header "barrier"
+    export *
+  }
+
+  module bit {
+    module bit_cast         { header "__bit/bit_cast.h" }
+    module bit_ceil         { header "__bit/bit_ceil.h" }
+    module bit_floor        { header "__bit/bit_floor.h" }
+    module bit_log2         { header "__bit/bit_log2.h" }
+    module bit_width        { header "__bit/bit_width.h" }
+    module blsr             { header "__bit/blsr.h" }
+    module byteswap         { header "__bit/byteswap.h" }
+    module countl           { header "__bit/countl.h" }
+    module countr           { header "__bit/countr.h" }
+    module endian           { header "__bit/endian.h" }
+    module has_single_bit   { header "__bit/has_single_bit.h" }
+    module invert_if        { header "__bit/invert_if.h" }
+    module popcount         { header "__bit/popcount.h" }
+    module rotate           { header "__bit/rotate.h" }
+
+    header "bit"
+    export *
+  }
+
+  module bitset {
+    header "bitset"
+    export *
+  }
+
+  module charconv {
+    module chars_format               { header "__charconv/chars_format.h" }
+    module from_chars_integral        { header "__charconv/from_chars_integral.h" }
+    module from_chars_result          { header "__charconv/from_chars_result.h" }
+    module tables                     { header "__charconv/tables.h" }
+    module to_chars                   { header "__charconv/to_chars.h" }
+    module to_chars_base_10           { header "__charconv/to_chars_base_10.h" }
+    module to_chars_floating_point    { header "__charconv/to_chars_floating_point.h" }
+    module to_chars_integral          { header "__charconv/to_chars_integral.h" }
+    module to_chars_result            { header "__charconv/to_chars_result.h" }
+    module traits                     { header "__charconv/traits.h" }
+
+    header "charconv"
+    export *
+  }
+
+  module chrono {
+    module calendar                   { header "__chrono/calendar.h" }
+    module concepts                   { header "__chrono/concepts.h" }
+    module convert_to_timespec        { header "__chrono/convert_to_timespec.h" }
+    module convert_to_tm              { header "__chrono/convert_to_tm.h" }
+    module day                        { header "__chrono/day.h" }
+    module duration                   { header "__chrono/duration.h" }
+    module exception                  { header "__chrono/exception.h" }
+    module file_clock                 { header "__chrono/file_clock.h" }
+    module formatter                  { header "__chrono/formatter.h" }
+    module hh_mm_ss                   { header "__chrono/hh_mm_ss.h" }
+    module high_resolution_clock {
+      header "__chrono/high_resolution_clock.h"
+      export *
+    }
+    module leap_second {
+      header "__chrono/leap_second.h"
+    }
+    module literals {
+      header "__chrono/literals.h"
+    }
+    module local_info {
+      header "__chrono/local_info.h"
+      export std.chrono.sys_info
+    }
+    module month_weekday              { header "__chrono/month_weekday.h" }
+    module month                      { header "__chrono/month.h" }
+    module monthday                   { header "__chrono/monthday.h" }
+    module ostream                    { header "__chrono/ostream.h" }
+    module parser_std_format_spec     { header "__chrono/parser_std_format_spec.h" }
+    module statically_widen           { header "__chrono/statically_widen.h" }
+    module steady_clock {
+      header "__chrono/steady_clock.h"
+      export std.chrono.time_point
+    }
+    module sys_info {
+      header "__chrono/sys_info.h"
+    }
+    module system_clock {
+      header "__chrono/system_clock.h"
+      export std.chrono.time_point
+    }
+    module time_point                 { header "__chrono/time_point.h" }
+    module time_zone_link             { header "__chrono/time_zone_link.h" }
+    module time_zone                  { header "__chrono/time_zone.h" }
+    module tzdb_list {
+      header "__chrono/tzdb_list.h"
+      export std.forward_list // forward_list iterators are used to implement this API
+      export std.string_view // by-value argument of type std::string_view
+    }
+    module tzdb {
+      header "__chrono/tzdb.h"
+      export std.string // public data member of type std::string
+      export std.vector // public data members of type std::vector
+    }
+    module weekday                    { header "__chrono/weekday.h" }
+    module year_month_day             { header "__chrono/year_month_day.h" }
+    module year_month_weekday         { header "__chrono/year_month_weekday.h" }
+    module year_month                 { header "__chrono/year_month.h" }
+    module year                       { header "__chrono/year.h" }
+    module zoned_time                 { header "__chrono/zoned_time.h" }
+
+    header "chrono"
+    export *
+  } // module chrono
+
+  module codecvt {
+    header "codecvt"
+    export *
+  }
+
+  module compare {
+    module common_comparison_category       { header "__compare/common_comparison_category.h" }
+    module compare_partial_order_fallback   { header "__compare/compare_partial_order_fallback.h" }
+    module compare_strong_order_fallback    { header "__compare/compare_strong_order_fallback.h" }
+    module compare_three_way                { header "__compare/compare_three_way.h" }
+    module compare_three_way_result         { header "__compare/compare_three_way_result.h" }
+    module compare_weak_order_fallback      { header "__compare/compare_weak_order_fallback.h" }
+    module is_eq                            { header "__compare/is_eq.h" }
+    module ordering                         { header "__compare/ordering.h" }
+    module partial_order                    { header "__compare/partial_order.h" }
+    module strong_order                     { header "__compare/strong_order.h" }
+    module synth_three_way                  { header "__compare/synth_three_way.h" }
+    module three_way_comparable             { header "__compare/three_way_comparable.h" }
+    module weak_order                       { header "__compare/weak_order.h" }
+
+    header "compare"
+    export *
+  }
+
+  module complex {
+    module fwd { header "__fwd/complex.h" }
+
+    header "complex"
+    export *
+  }
+
+  module concepts {
+    module arithmetic               { header "__concepts/arithmetic.h" }
+    module assignable               { header "__concepts/assignable.h" }
+    module boolean_testable         { header "__concepts/boolean_testable.h" }
+    module class_or_enum            { header "__concepts/class_or_enum.h" }
+    module common_reference_with    { header "__concepts/common_reference_with.h" }
+    module common_with              { header "__concepts/common_with.h" }
+    module constructible            { header "__concepts/constructible.h" }
+    module convertible_to           { header "__concepts/convertible_to.h" }
+    module copyable                 { header "__concepts/copyable.h" }
+    module derived_from             { header "__concepts/derived_from.h" }
+    module destructible             { header "__concepts/destructible.h" }
+    module different_from           { header "__concepts/different_from.h" }
+    module equality_comparable      { header "__concepts/equality_comparable.h" }
+    module invocable                { header "__concepts/invocable.h" }
+    module movable                  { header "__concepts/movable.h" }
+    module predicate                { header "__concepts/predicate.h" }
+    module regular                  { header "__concepts/regular.h" }
+    module relation                 { header "__concepts/relation.h" }
+    module same_as                  { header "__concepts/same_as.h" }
+    module semiregular              { header "__concepts/semiregular.h" }
+    module swappable                { header "__concepts/swappable.h" }
+    module totally_ordered          { header "__concepts/totally_ordered.h" }
+
+    header "concepts"
+    export *
+  }
+
+  module condition_variable {
+    module condition_variable   { header "__condition_variable/condition_variable.h" }
+
+    header "condition_variable"
+    export *
+  }
+
+  module cassert {
+    textual header "cassert" // NDEBUG requires textual inclusion
+  }
+
+  module ccomplex {
+    header "ccomplex"
+    export *
+  }
+
+  module cctype {
+    header "cctype"
+    export *
+  }
+
+  module cerrno {
+    header "cerrno"
+    export *
+  }
+
+  module cfenv {
+    header "cfenv"
+    export *
+  }
+
+  module cfloat {
+    header "cfloat"
+    export *
+  }
+
+  module cinttypes {
+    header "cinttypes"
+    export *
+  }
+
+  module ciso646 {
+    header "ciso646"
+    export *
+  }
+
+  module climits {
+    header "climits"
+    export *
+  }
+
+  module clocale {
+    header "clocale"
+    export *
+  }
+
+  module cmath {
+    header "cmath"
+    export *
+  }
+
+  // TODO: Make non-textual. This seems to cause problems when compiling against Glibc.
+  module csetjmp {
+    textual header "csetjmp"
+  }
+
+  module csignal {
+    header "csignal"
+    export *
+  }
+
+  module cstdarg {
+    header "cstdarg"
+    export *
+  }
+
+  module cstdbool {
+    header "cstdbool"
+    export *
+  }
+
+  module cstddef {
+    header "cstddef"
+    export *
+  }
+
+  module cstdio {
+    header "cstdio"
+    export *
+  }
+
+  module cstdlib {
+    header "cstdlib"
+    export *
+  }
+
+  module cstring {
+    header "cstring"
+    export *
+  }
+
+  module ctgmath {
+    header "ctgmath"
+    export *
+  }
+
+  module ctime {
+    header "ctime"
+    export *
+  }
+
+  module cuchar {
+    header "cuchar"
+    export *
+  }
+
+  module cwchar {
+    header "cwchar"
+    export *
+  }
+
+  module cwctype {
+    header "cwctype"
+    export *
+  }
+
+  module deque {
+    module fwd { header "__fwd/deque.h" }
+
+    header "deque"
+    export *
+  }
+
+  module exception {
+    module exception          { header "__exception/exception.h" }
+    module exception_ptr      { header "__exception/exception_ptr.h" }
+    module nested_exception   { header "__exception/nested_exception.h" }
+    module operations         { header "__exception/operations.h" }
+    module terminate          { header "__exception/terminate.h" }
+
+    header "exception"
+    export *
+  }
+
+  module execution {
+    header "execution"
+    export *
+  }
+
+  module expected {
+    module bad_expected_access    { header "__expected/bad_expected_access.h" }
+    module expected               { header "__expected/expected.h" }
+    module unexpect               { header "__expected/unexpect.h" }
+    module unexpected             { header "__expected/unexpected.h" }
+
+    header "expected"
+    export *
+  }
+
+  module filesystem {
+    module copy_options                   { header "__filesystem/copy_options.h" }
+    module directory_entry                { header "__filesystem/directory_entry.h" }
+    module directory_iterator             { header "__filesystem/directory_iterator.h" }
+    module directory_options              { header "__filesystem/directory_options.h" }
+    module file_status                    { header "__filesystem/file_status.h" }
+    module file_time_type                 { header "__filesystem/file_time_type.h" }
+    module file_type                      { header "__filesystem/file_type.h" }
+    module filesystem_error               { header "__filesystem/filesystem_error.h" }
+    module operations                     { header "__filesystem/operations.h" }
+    module path_iterator                  { header "__filesystem/path_iterator.h" }
+    module path                           {
+      header "__filesystem/path.h"
+      export std.string // returned by various methods of filesystem::path
+    }
+    module perm_options                   { header "__filesystem/perm_options.h" }
+    module perms                          { header "__filesystem/perms.h" }
+    module recursive_directory_iterator   { header "__filesystem/recursive_directory_iterator.h" }
+    module space_info                     { header "__filesystem/space_info.h" }
+    module u8path                         { header "__filesystem/u8path.h" }
+
+    header "filesystem"
+    export *
+  }
+
+  module format {
+    module buffer                             { header "__format/buffer.h" }
+    module concepts                           { header "__format/concepts.h" }
+    module container_adaptor                  { header "__format/container_adaptor.h" }
+    module enable_insertable                  { header "__format/enable_insertable.h" }
+    module escaped_output_table               { header "__format/escaped_output_table.h" }
+    module extended_grapheme_cluster_table    { header "__format/extended_grapheme_cluster_table.h" }
+    module format_arg                         { header "__format/format_arg.h" }
+    module format_arg_store                   { header "__format/format_arg_store.h" }
+    module format_args                        { header "__format/format_args.h" }
+    module format_context {
+      header "__format/format_context.h"
+      export std.optional // default argument for __format_context_create
+    }
+    module format_error {
+      header "__format/format_error.h"
+    }
+    module format_functions {
+      header "__format/format_functions.h"
+      export std.string // returned by the functions in that header
+    }
+    module format_parse_context               { header "__format/format_parse_context.h" }
+    module format_string                      { header "__format/format_string.h" }
+    module format_to_n_result                 { header "__format/format_to_n_result.h" }
+    module formatter                          { header "__format/formatter.h" }
+    module formatter_bool                     { header "__format/formatter_bool.h" }
+    module formatter_char                     { header "__format/formatter_char.h" }
+    module formatter_floating_point           { header "__format/formatter_floating_point.h" }
+    module formatter_integer                  { header "__format/formatter_integer.h" }
+    module formatter_integral                 { header "__format/formatter_integral.h" }
+    module formatter_output                   { header "__format/formatter_output.h" }
+    module formatter_pointer                  { header "__format/formatter_pointer.h" }
+    module formatter_string                   { header "__format/formatter_string.h" }
+    module formatter_tuple                    { header "__format/formatter_tuple.h" }
+    module fwd                                { header "__fwd/format.h" }
+    module indic_conjunct_break_table         { header "__format/indic_conjunct_break_table.h" }
+    module parser_std_format_spec             { header "__format/parser_std_format_spec.h" }
+    module range_default_formatter            { header "__format/range_default_formatter.h" }
+    module range_formatter                    { header "__format/range_formatter.h" }
+    module unicode                            { header "__format/unicode.h" }
+    module width_estimation_table             { header "__format/width_estimation_table.h" }
+    module write_escaped                      { header "__format/write_escaped.h" }
+
+    header "format"
+    export *
+  } // module format
+
+  module forward_list {
+    header "forward_list"
+    export *
+  }
+
+  module fstream {
+    module fwd { header "__fwd/fstream.h" }
+
+    header "fstream"
+    export *
+  }
+
+  module functional {
+    module binary_function              { header "__functional/binary_function.h" }
+    module binary_negate                { header "__functional/binary_negate.h" }
+    module bind_back {
+      header "__functional/bind_back.h"
+      export std.functional.perfect_forward // inherited from and using its operators
+    }
+    module bind_front {
+      header "__functional/bind_front.h"
+      export std.functional.perfect_forward // inherited from and using its operators
+    }
+    module bind                         { header "__functional/bind.h" }
+    module binder1st                    { header "__functional/binder1st.h" }
+    module binder2nd                    { header "__functional/binder2nd.h" }
+    module boyer_moore_searcher {
+      header "__functional/boyer_moore_searcher.h"
+      export std.memory.shared_ptr
+    }
+    module compose {
+      header "__functional/compose.h"
+      export std.functional.perfect_forward // inherited from and using its operators
+    }
+    module default_searcher             { header "__functional/default_searcher.h" }
+    module function                     { header "__functional/function.h" }
+    module hash                         { header "__functional/hash.h" }
+    module identity                     { header "__functional/identity.h" }
+    module invoke                       { header "__functional/invoke.h" }
+    module is_transparent               { header "__functional/is_transparent.h" }
+    module mem_fn                       { header "__functional/mem_fn.h" }
+    module mem_fun_ref                  { header "__functional/mem_fun_ref.h" }
+    module not_fn {
+      header "__functional/not_fn.h"
+      export std.functional.perfect_forward // inherited from and using its operators
+    }
+    module operations                   { header "__functional/operations.h" }
+    module perfect_forward {
+      header "__functional/perfect_forward.h"
+      export std.tuple
+    }
+    module pointer_to_binary_function   { header "__functional/pointer_to_binary_function.h" }
+    module pointer_to_unary_function    { header "__functional/pointer_to_unary_function.h" }
+    module ranges_operations            { header "__functional/ranges_operations.h" }
+    module reference_wrapper            { header "__functional/reference_wrapper.h" }
+    module unary_function               { header "__functional/unary_function.h" }
+    module unary_negate                 { header "__functional/unary_negate.h" }
+    module weak_result_type             { header "__functional/weak_result_type.h" }
+
+    header "functional"
+    export *
+  } // module functional
+
+  module future {
+    header "future"
+    export *
+  }
+
+  module initializer_list {
+    header "initializer_list"
+    export *
+  }
+
+  module iomanip {
+    header "iomanip"
+    export *
+  }
+
+  module ios {
+    module fwd  { header "__fwd/ios.h" }
+    module fpos { header "__ios/fpos.h" }
+
+    header "ios"
+    export *
+  }
+
+  module iosfwd {
+    header "iosfwd"
+    export *
+  }
+
+  module iostream {
+    header "iostream"
+    export *
+  }
+
+  module istream {
+    module fwd { header "__fwd/istream.h" }
+
+    header "istream"
+    export std.ios // base class
+  }
 
-// Experimental C++ standard library interfaces
-module std_experimental [system] {
   module iterator {
-    header "experimental/iterator"
+    module access                     { header "__iterator/access.h" }
+    module advance                    { header "__iterator/advance.h" }
+    module aliasing_iterator          { header "__iterator/aliasing_iterator.h" }
+    module back_insert_iterator       { header "__iterator/back_insert_iterator.h" }
+    module bounded_iter               { header "__iterator/bounded_iter.h" }
+    module common_iterator            { header "__iterator/common_iterator.h" }
+    module concepts {
+      header "__iterator/concepts.h"
+      export std_core.type_traits.common_reference
+    }
+    module counted_iterator           { header "__iterator/counted_iterator.h" }
+    module cpp17_iterator_concepts    { header "__iterator/cpp17_iterator_concepts.h" }
+    module data                       { header "__iterator/data.h" }
+    module default_sentinel           { header "__iterator/default_sentinel.h" }
+    module distance                   { header "__iterator/distance.h" }
+    module empty                      { header "__iterator/empty.h" }
+    module erase_if_container         { header "__iterator/erase_if_container.h" }
+    module front_insert_iterator      { header "__iterator/front_insert_iterator.h" }
+    module incrementable_traits       { header "__iterator/incrementable_traits.h" }
+    module indirectly_comparable      { header "__iterator/indirectly_comparable.h" }
+    module insert_iterator            { header "__iterator/insert_iterator.h" }
+    module istream_iterator           { header "__iterator/istream_iterator.h" }
+    module istreambuf_iterator        { header "__iterator/istreambuf_iterator.h" }
+    module iter_move                  { header "__iterator/iter_move.h" }
+    module iter_swap                  { header "__iterator/iter_swap.h" }
+    module iterator_traits {
+      header "__iterator/iterator_traits.h"
+      export std_core.type_traits.integral_constant
+    }
+    module iterator_with_data         { header "__iterator/iterator_with_data.h" }
+    module iterator                   { header "__iterator/iterator.h" }
+    module mergeable                  { header "__iterator/mergeable.h" }
+    module move_iterator              { header "__iterator/move_iterator.h" }
+    module move_sentinel              { header "__iterator/move_sentinel.h" }
+    module next                       { header "__iterator/next.h" }
+    module ostream_iterator           { header "__iterator/ostream_iterator.h" }
+    module ostreambuf_iterator {
+      header "__iterator/ostreambuf_iterator.h"
+      export iosfwd // for default template argument of ostreambuf_iterator
+    }
+    module permutable                 { header "__iterator/permutable.h" }
+    module prev                       { header "__iterator/prev.h" }
+    module projected                  { header "__iterator/projected.h" }
+    module ranges_iterator_traits     { header "__iterator/ranges_iterator_traits.h" }
+    module readable_traits            { header "__iterator/readable_traits.h" }
+    module reverse_access             { header "__iterator/reverse_access.h" }
+    module reverse_iterator           { header "__iterator/reverse_iterator.h" }
+    module segmented_iterator         { header "__iterator/segmented_iterator.h" }
+    module size                       { header "__iterator/size.h" }
+    module sortable                   { header "__iterator/sortable.h" }
+    module unreachable_sentinel       { header "__iterator/unreachable_sentinel.h" }
+    module wrap_iter                  { header "__iterator/wrap_iter.h" }
+
+    header "iterator"
+    export *
+  }
+
+  module latch {
+    header "latch"
+    export *
+  }
+
+  module list {
+    header "list"
+    export *
+  }
+
+  module locale {
+    header "locale"
+    header "__locale_dir/locale_base_api.h"
+    header "__locale_dir/locale_base_api/locale_guard.h"
+    module locale_base_api {
+      textual header "__locale_dir/locale_base_api/android.h"
+      textual header "__locale_dir/locale_base_api/bsd_locale_defaults.h"
+      textual header "__locale_dir/locale_base_api/bsd_locale_fallbacks.h"
+      textual header "__locale_dir/locale_base_api/fuchsia.h"
+      textual header "__locale_dir/locale_base_api/ibm.h"
+      textual header "__locale_dir/locale_base_api/musl.h"
+      textual header "__locale_dir/locale_base_api/newlib.h"
+      textual header "__locale_dir/locale_base_api/openbsd.h"
+      textual header "__locale_dir/locale_base_api/win32.h"
+    }
+    export *
+  }
+
+  // TODO: Understand why this needs to live in its own module
+  module locale_base [system] {
+    header "__locale"
+    export *
+  }
+
+  module map {
+    header "map"
+    export *
+  }
+
+  module mdspan {
+    module default_accessor   { header "__mdspan/default_accessor.h" }
+    module extents            { header "__mdspan/extents.h" }
+    module fwd                { header "__fwd/mdspan.h" }
+    module layout_left        { header "__mdspan/layout_left.h" }
+    module layout_right       { header "__mdspan/layout_right.h" }
+    module layout_stride      { header "__mdspan/layout_stride.h" }
+    module mdspan {
+      header "__mdspan/mdspan.h"
+      export std.array // returned by some methods
+    }
+
+    header "mdspan"
     export *
   }
+
   module memory {
-    header "experimental/memory"
+    module addressof                          { header "__memory/addressof.h" }
+    module align                              { header "__memory/align.h" }
+    module aligned_alloc                      { header "__memory/aligned_alloc.h" }
+    module allocate_at_least                  { header "__memory/allocate_at_least.h" }
+    module allocation_guard                   { header "__memory/allocation_guard.h" }
+    module allocator                          { header "__memory/allocator.h" }
+    module allocator_arg_t                    { header "__memory/allocator_arg_t.h" }
+    module allocator_destructor               { header "__memory/allocator_destructor.h" }
+    module allocator_traits                   { header "__memory/allocator_traits.h" }
+    module assume_aligned                     { header "__memory/assume_aligned.h" }
+    module array_cookie                       { header "__memory/array_cookie.h" }
+    module auto_ptr                           { header "__memory/auto_ptr.h" }
+    module builtin_new_allocator              { header "__memory/builtin_new_allocator.h" }
+    module compressed_pair                    { header "__memory/compressed_pair.h" }
+    module concepts                           { header "__memory/concepts.h" }
+    module construct_at                       { header "__memory/construct_at.h" }
+    module destruct_n                         { header "__memory/destruct_n.h" }
+    module fwd                                { header "__fwd/memory.h" }
+    module inout_ptr                          { header "__memory/inout_ptr.h" }
+    module noexcept_move_assign_container     { header "__memory/noexcept_move_assign_container.h" }
+    module out_ptr                            { header "__memory/out_ptr.h" }
+    module pointer_traits                     { header "__memory/pointer_traits.h" }
+    module ranges_construct_at                { header "__memory/ranges_construct_at.h" }
+    module ranges_uninitialized_algorithms {
+      header "__memory/ranges_uninitialized_algorithms.h"
+      export std.algorithm.in_out_result
+    }
+    module raw_storage_iterator               { header "__memory/raw_storage_iterator.h" }
+    module shared_ptr                         { header "__memory/shared_ptr.h" }
+    module swap_allocator                     { header "__memory/swap_allocator.h" }
+    module temp_value                         { header "__memory/temp_value.h" }
+    module temporary_buffer                   {
+      header "__memory/temporary_buffer.h"
+      export std.utility.pair // return type of std::get_temporary_buffer()
+    }
+    module uninitialized_algorithms {
+      header "__memory/uninitialized_algorithms.h"
+    }
+    module unique_ptr {
+      header "__memory/unique_ptr.h"
+    }
+    module unique_temporary_buffer {
+      header "__memory/unique_temporary_buffer.h"
+      export std.memory.unique_ptr
+      export std_core.type_traits.is_constant_evaluated
+    }
+    module uses_allocator                     { header "__memory/uses_allocator.h" }
+    module uses_allocator_construction        { header "__memory/uses_allocator_construction.h" }
+    module voidify                            { header "__memory/voidify.h" }
+
+    header "memory"
     export *
   }
-  module propagate_const {
-    header "experimental/propagate_const"
+
+  module memory_resource {
+    module fwd                            { header "__fwd/memory_resource.h" }
+    module memory_resource                { header "__memory_resource/memory_resource.h" }
+    module monotonic_buffer_resource      { header "__memory_resource/monotonic_buffer_resource.h" }
+    module polymorphic_allocator          { header "__memory_resource/polymorphic_allocator.h" }
+    module pool_options                   { header "__memory_resource/pool_options.h" }
+    module synchronized_pool_resource     { header "__memory_resource/synchronized_pool_resource.h" }
+    module unsynchronized_pool_resource   { header "__memory_resource/unsynchronized_pool_resource.h" }
+
+    header "memory_resource"
     export *
   }
-  module simd {
-    module aligned_tag          { private header "experimental/__simd/aligned_tag.h" }
-    module declaration          { private header "experimental/__simd/declaration.h" }
-    module reference            { private header "experimental/__simd/reference.h" }
-    module scalar               { private header "experimental/__simd/scalar.h" }
-    module simd                 { private header "experimental/__simd/simd.h" }
-    module simd_mask            { private header "experimental/__simd/simd_mask.h" }
-    module traits               { private header "experimental/__simd/traits.h" }
-    module utility              { private header "experimental/__simd/utility.h" }
-    module vec_ext              { private header "experimental/__simd/vec_ext.h" }
 
-    header "experimental/simd"
+  module mutex {
+    module lock_guard     { header "__mutex/lock_guard.h" }
+    module mutex          { header "__mutex/mutex.h" }
+    module once_flag      { header "__mutex/once_flag.h" }
+    module tag_types      { header "__mutex/tag_types.h" }
+    module unique_lock    { header "__mutex/unique_lock.h" }
+
+    header "mutex"
     export *
   }
-  module type_traits {
-    header "experimental/type_traits"
+
+  module new {
+    header "new"
     export *
   }
-  module utility {
-    header "experimental/utility"
+
+  module numbers {
+    header "numbers"
     export *
   }
-}
 
-// Convenience method to get all of the above modules in a single import statement.
-// Importing only the needed modules is likely to be more performant.
-module std [system] {
-  header "__std_clang_module"
-  export *
-}
+  module numeric {
+    module accumulate                 { header "__numeric/accumulate.h" }
+    module adjacent_difference        { header "__numeric/adjacent_difference.h" }
+    module exclusive_scan             { header "__numeric/exclusive_scan.h" }
+    module gcd_lcm                    { header "__numeric/gcd_lcm.h" }
+    module inclusive_scan             { header "__numeric/inclusive_scan.h" }
+    module inner_product              { header "__numeric/inner_product.h" }
+    module iota                       { header "__numeric/iota.h" }
+    module midpoint                   { header "__numeric/midpoint.h" }
+    module partial_sum                { header "__numeric/partial_sum.h" }
+    module pstl                       { header "__numeric/pstl.h" }
+    module reduce                     { header "__numeric/reduce.h" }
+    module saturation_arithmetic      { header "__numeric/saturation_arithmetic.h" }
+    module transform_exclusive_scan   { header "__numeric/transform_exclusive_scan.h" }
+    module transform_inclusive_scan   { header "__numeric/transform_inclusive_scan.h" }
+    module transform_reduce           { header "__numeric/transform_reduce.h" }
 
-// Implementation detail headers that are private to libc++. These modules
-// must not be directly imported.
-module std_private_assert            [system] {
-  header "__assert"
-  export *
-}
-module std_private_bit_reference     [system] {
-  header "__bit_reference"
-  export *
-}
-module std_private_fwd_bit_reference [system] {
-  header "__fwd/bit_reference.h"
-}
-module std_private_fwd_byte [system] {
-  header "__fwd/byte.h"
-}
-module std_private_config            [system] {
-  textual header "__config"
-  textual header "__configuration/abi.h"
-  textual header "__configuration/availability.h"
-  textual header "__configuration/compiler.h"
-  textual header "__configuration/language.h"
-  textual header "__configuration/platform.h"
-  export *
-}
-module std_private_hash_table        [system] {
-  header "__hash_table"
-  export *
-}
-module std_private_locale            [system] {
-  header "__locale"
-  export *
-}
-module std_private_mbstate_t         [system] {
-  header "__mbstate_t.h"
-  export *
-}
-module std_private_node_handle       [system] {
-  header "__node_handle"
-  export *
-}
-module std_private_split_buffer      [system] {
-  header "__split_buffer"
-  export *
-}
-module std_private_std_mbstate_t     [system] {
-  header "__std_mbstate_t.h"
-  export *
-}
-module std_private_tree              [system] {
-  header "__tree"
-  export *
-}
-module std_private_undef_macros      [system] {
-  textual header "__undef_macros"
-  export *
-}
-module std_private_verbose_abort     [system] {
-  header "__verbose_abort"
-  export *
-}
+    header "numeric"
+    export *
+  }
+
+  module optional {
+    header "optional"
+    export *
+  }
+
+  module ostream {
+    module basic_ostream {
+      header "__ostream/basic_ostream.h"
+      export std.ios // base class
+    }
+    module fwd {
+      header "__fwd/ostream.h"
+    }
+    module print {
+      header "__ostream/print.h"
+      export *
+    }
+
+    header "ostream"
+    export *
+  }
+
+  module print {
+    header "print"
+    export *
+  }
+
+  module queue {
+    module fwd { header "__fwd/queue.h" }
+
+    header "queue"
+    export *
+  }
+
+  module random {
+    module bernoulli_distribution             { header "__random/bernoulli_distribution.h" }
+    module binomial_distribution              { header "__random/binomial_distribution.h" }
+    module cauchy_distribution                { header "__random/cauchy_distribution.h" }
+    module chi_squared_distribution           { header "__random/chi_squared_distribution.h" }
+    module clamp_to_integral                  { header "__random/clamp_to_integral.h" }
+    module default_random_engine              { header "__random/default_random_engine.h" }
+    module discard_block_engine               { header "__random/discard_block_engine.h" }
+    module discrete_distribution              { header "__random/discrete_distribution.h" }
+    module exponential_distribution           { header "__random/exponential_distribution.h" }
+    module extreme_value_distribution         { header "__random/extreme_value_distribution.h" }
+    module fisher_f_distribution              { header "__random/fisher_f_distribution.h" }
+    module gamma_distribution                 { header "__random/gamma_distribution.h" }
+    module generate_canonical                 { header "__random/generate_canonical.h" }
+    module geometric_distribution             { header "__random/geometric_distribution.h" }
+    module independent_bits_engine            { header "__random/independent_bits_engine.h" }
+    module is_seed_sequence                   { header "__random/is_seed_sequence.h" }
+    module is_valid {
+      header "__random/is_valid.h"
+      export std_core.type_traits.integral_constant
+    }
+    module knuth_b                            { header "__random/knuth_b.h" }
+    module linear_congruential_engine         { header "__random/linear_congruential_engine.h" }
+    module log2                               { header "__random/log2.h" }
+    module lognormal_distribution             { header "__random/lognormal_distribution.h" }
+    module mersenne_twister_engine            { header "__random/mersenne_twister_engine.h" }
+    module negative_binomial_distribution     { header "__random/negative_binomial_distribution.h" }
+    module normal_distribution                { header "__random/normal_distribution.h" }
+    module piecewise_constant_distribution    { header "__random/piecewise_constant_distribution.h" }
+    module piecewise_linear_distribution      { header "__random/piecewise_linear_distribution.h" }
+    module poisson_distribution               { header "__random/poisson_distribution.h" }
+    module random_device                      { header "__random/random_device.h" }
+    module ranlux                             { header "__random/ranlux.h" }
+    module seed_seq                           { header "__random/seed_seq.h" }
+    module shuffle_order_engine               { header "__random/shuffle_order_engine.h" }
+    module student_t_distribution             { header "__random/student_t_distribution.h" }
+    module subtract_with_carry_engine         { header "__random/subtract_with_carry_engine.h" }
+    module uniform_int_distribution           { header "__random/uniform_int_distribution.h" }
+    module uniform_random_bit_generator       { header "__random/uniform_random_bit_generator.h" }
+    module uniform_real_distribution          { header "__random/uniform_real_distribution.h" }
+    module weibull_distribution               { header "__random/weibull_distribution.h" }
+
+    header "random"
+    export *
+  }
+
+  module ranges {
+    module access                         { header "__ranges/access.h" }
+    module all                            { header "__ranges/all.h" }
+    module as_rvalue_view                 { header "__ranges/as_rvalue_view.h" }
+    module chunk_by_view {
+      header "__ranges/chunk_by_view.h"
+      export std.functional.bind_back
+    }
+    module common_view                    { header "__ranges/common_view.h" }
+    module concepts                       { header "__ranges/concepts.h" }
+    module container_compatible_range     { header "__ranges/container_compatible_range.h" }
+    module counted {
+      header "__ranges/counted.h"
+      export std.span            // return type of views::counted
+      export std.ranges.subrange // return type of views::counted
+    }
+    module dangling {
+      header "__ranges/dangling.h"
+    }
+    module data {
+      header "__ranges/data.h"
+    }
+    module drop_view {
+      header "__ranges/drop_view.h"
+      export std.functional.bind_back
+    }
+    module drop_while_view {
+      header "__ranges/drop_while_view.h"
+      export std.functional.bind_back
+    }
+    module elements_view                  { header "__ranges/elements_view.h" }
+    module empty                          { header "__ranges/empty.h" }
+    module empty_view                     { header "__ranges/empty_view.h" }
+    module enable_borrowed_range          { header "__ranges/enable_borrowed_range.h" }
+    module enable_view                    { header "__ranges/enable_view.h" }
+    module filter_view {
+      header "__ranges/filter_view.h"
+      export std.functional.bind_back
+    }
+    module from_range                     { header "__ranges/from_range.h" }
+    module iota_view                      { header "__ranges/iota_view.h" }
+    module istream_view                   { header "__ranges/istream_view.h" }
+    module join_view                      { header "__ranges/join_view.h" }
+    module lazy_split_view {
+      header "__ranges/lazy_split_view.h"
+      export std.functional.bind_back
+    }
+    module movable_box                    { header "__ranges/movable_box.h" }
+    module non_propagating_cache          { header "__ranges/non_propagating_cache.h" }
+    module owning_view                    { header "__ranges/owning_view.h" }
+    module range_adaptor                  { header "__ranges/range_adaptor.h" }
+    module rbegin                         { header "__ranges/rbegin.h" }
+    module ref_view                       { header "__ranges/ref_view.h" }
+    module rend                           { header "__ranges/rend.h" }
+    module repeat_view                    { header "__ranges/repeat_view.h" }
+    module reverse_view                   { header "__ranges/reverse_view.h" }
+    module single_view                    { header "__ranges/single_view.h" }
+    module size                           { header "__ranges/size.h" }
+    module split_view {
+      header "__ranges/split_view.h"
+      export std.functional.bind_back
+    }
+    module subrange {
+      header "__ranges/subrange.h"
+      export std.ranges.subrange_fwd
+    }
+    module subrange_fwd {
+      header "__fwd/subrange.h"
+    }
+    module take_view {
+      header "__ranges/take_view.h"
+      export std.functional.bind_back
+    }
+    module take_while_view {
+      header "__ranges/take_while_view.h"
+      export std.functional.bind_back
+    }
+    module to {
+      header "__ranges/to.h"
+      export std.functional.bind_back
+    }
+    module transform_view {
+      header "__ranges/transform_view.h"
+      export std.functional.bind_back
+    }
+    module view_interface {
+      header "__ranges/view_interface.h"
+    }
+    module views {
+      header "__ranges/views.h"
+    }
+    module zip_view {
+      header "__ranges/zip_view.h"
+      export std.utility.pair
+    }
+
+    header "ranges"
+    export *
+  } // module ranges
+
+  module ratio {
+    header "ratio"
+    export *
+  }
+
+  module regex {
+    header "regex"
+    export *
+  }
+
+  module scoped_allocator {
+    header "scoped_allocator"
+    export *
+  }
+
+  module semaphore {
+    header "semaphore"
+    export *
+  }
+
+  module set {
+    header "set"
+    export *
+  }
+
+  module shared_mutex {
+    header "shared_mutex"
+    export *
+  }
+
+  module source_location {
+    header "source_location"
+    export *
+  }
+
+  module span {
+    module fwd { header "__fwd/span.h" }
+
+    header "span"
+    export *
+  }
+
+  module sstream {
+    module fwd { header "__fwd/sstream.h" }
+
+    header "sstream"
+    export *
+  }
+
+  module stack {
+    module fwd { header "__fwd/stack.h" }
+
+    header "stack"
+    export *
+  }
+
+  module stdexcept {
+    header "stdexcept"
+    export *
+  }
+
+  module stop_token {
+    module atomic_unique_lock     { header "__stop_token/atomic_unique_lock.h" }
+    module intrusive_list_view    { header "__stop_token/intrusive_list_view.h" }
+    module intrusive_shared_ptr   { header "__stop_token/intrusive_shared_ptr.h" }
+    module stop_callback          { header "__stop_token/stop_callback.h" }
+    module stop_source            { header "__stop_token/stop_source.h" }
+    module stop_state             { header "__stop_token/stop_state.h" }
+    module stop_token             { header "__stop_token/stop_token.h" }
+
+    header "stop_token"
+    export *
+  }
+
+  module streambuf {
+    module fwd { header "__fwd/streambuf.h" }
+
+    header "streambuf"
+    export *
+  }
+
+  module string {
+    module char_traits              { header "__string/char_traits.h" }
+    module constexpr_c_functions    { header "__string/constexpr_c_functions.h" }
+    module extern_template_lists    { header "__string/extern_template_lists.h" }
+    module fwd                      {  header "__fwd/string.h" }
+
+    header "string"
+    export *
+  }
+
+  module string_view {
+    module fwd { header "__fwd/string_view.h" }
+
+    header "string_view"
+    export *
+  }
+
+  module strstream {
+    header "strstream"
+    export *
+  }
+
+  module syncstream {
+    header "syncstream"
+    export *
+  }
+
+  module system_error {
+    module errc               { header "__system_error/errc.h" }
+    module error_category     { header "__system_error/error_category.h" }
+    module error_code {
+      header "__system_error/error_code.h"
+      export std.system_error.error_category // methods of error_code return that type
+    }
+    module error_condition    { header "__system_error/error_condition.h" }
+    module system_error       { header "__system_error/system_error.h" }
+
+    header "system_error"
+    export *
+  }
+
+  module thread {
+    module formatter              { header "__thread/formatter.h" }
+    module id                     { header "__thread/id.h" }
+    module jthread                { header "__thread/jthread.h" }
+    module poll_with_backoff      { header "__thread/poll_with_backoff.h" }
+    module this_thread            { header "__thread/this_thread.h" }
+    module thread                 { header "__thread/thread.h" }
+    module timed_backoff_policy   { header "__thread/timed_backoff_policy.h" }
+
+    module support {
+      header "__thread/support.h"
+      export *
+    }
+    module support_impl {
+      textual header "__thread/support/c11.h"
+      textual header "__thread/support/external.h"
+      textual header "__thread/support/pthread.h"
+      textual header "__thread/support/windows.h"
+    }
+
+    header "thread"
+    export *
+  }
+
+  module tuple {
+    module find_index               { header "__tuple/find_index.h" }
+    module ignore                   { header "__tuple/ignore.h" }
+    module make_tuple_types         { header "__tuple/make_tuple_types.h" }
+    module sfinae_helpers           { header "__tuple/sfinae_helpers.h" }
+    module tuple_element            { header "__tuple/tuple_element.h" }
+    module tuple_indices            { header "__tuple/tuple_indices.h" }
+    module tuple_like_ext           { header "__tuple/tuple_like_ext.h" }
+    module tuple_like_no_subrange   { header "__tuple/tuple_like_no_subrange.h" }
+    module tuple_like               { header "__tuple/tuple_like.h" }
+    module tuple_size               { header "__tuple/tuple_size.h" }
+    module tuple_types              { header "__tuple/tuple_types.h" }
+
+    header "tuple"
+    export *
+  }
+
+  module typeindex {
+    header "typeindex"
+    export *
+  }
+
+  module typeinfo {
+    header "typeinfo"
+    export *
+  }
+
+  module unordered_map {
+    header "unordered_map"
+    export *
+  }
+
+  module unordered_set {
+    header "unordered_set"
+    export *
+  }
+
+  module utility {
+    module as_const                   { header "__utility/as_const.h" }
+    module as_lvalue                  { header "__utility/as_lvalue.h" }
+    module auto_cast                  {
+      header "__utility/auto_cast.h"
+      export std_core.type_traits.decay // the macro expansion uses that trait
+    }
+    module cmp                        { header "__utility/cmp.h" }
+    module convert_to_integral        { header "__utility/convert_to_integral.h" }
+    module exception_guard            { header "__utility/exception_guard.h" }
+    module exchange                   { header "__utility/exchange.h" }
+    module forward_like               { header "__utility/forward_like.h" }
+    module in_place {
+      header "__utility/in_place.h"
+      export std_core.type_traits.integral_constant
+    }
+    module integer_sequence           { header "__utility/integer_sequence.h" }
+    module is_pointer_in_range        { header "__utility/is_pointer_in_range.h" }
+    module is_valid_range             { header "__utility/is_valid_range.h" }
+    module move                       { header "__utility/move.h" }
+    module no_destroy                 { header "__utility/no_destroy.h" }
+    module pair                       { header "__utility/pair.h" }
+    module piecewise_construct        { header "__utility/piecewise_construct.h" }
+    module priority_tag               { header "__utility/priority_tag.h" }
+    module private_constructor_tag    { header "__utility/private_constructor_tag.h" }
+    module rel_ops                    { header "__utility/rel_ops.h" }
+    module small_buffer               { header "__utility/small_buffer.h" }
+    module swap                       { header "__utility/swap.h" }
+    module to_underlying              { header "__utility/to_underlying.h" }
+    module unreachable                { header "__utility/unreachable.h" }
+
+    header "utility"
+    export *
+  }
+
+  module valarray {
+    header "valarray"
+    export *
+  }
 
-module std_private_algorithm_adjacent_find                               [system] { header "__algorithm/adjacent_find.h" }
-module std_private_algorithm_all_of                                      [system] { header "__algorithm/all_of.h" }
-module std_private_algorithm_any_of                                      [system] { header "__algorithm/any_of.h" }
-module std_private_algorithm_binary_search                               [system] { header "__algorithm/binary_search.h" }
-module std_private_algorithm_clamp                                       [system] { header "__algorithm/clamp.h" }
-module std_private_algorithm_comp                                        [system] { header "__algorithm/comp.h" }
-module std_private_algorithm_comp_ref_type                               [system] { header "__algorithm/comp_ref_type.h" }
-module std_private_algorithm_copy                                        [system] {
-  header "__algorithm/copy.h"
-  export std_private_algorithm_copy_move_common
-}
-module std_private_algorithm_copy_backward                               [system] { header "__algorithm/copy_backward.h" }
-module std_private_algorithm_copy_if                                     [system] { header "__algorithm/copy_if.h" }
-module std_private_algorithm_copy_move_common                            [system] {
-  header "__algorithm/copy_move_common.h"
-  export std_private_type_traits_is_trivially_copyable
-}
-module std_private_algorithm_copy_n                                      [system] { header "__algorithm/copy_n.h" }
-module std_private_algorithm_count                                       [system] { header "__algorithm/count.h" }
-module std_private_algorithm_count_if                                    [system] { header "__algorithm/count_if.h" }
-module std_private_algorithm_equal                                       [system] { header "__algorithm/equal.h" }
-module std_private_algorithm_equal_range                                 [system] { header "__algorithm/equal_range.h" }
-module std_private_algorithm_fill                                        [system] { header "__algorithm/fill.h" }
-module std_private_algorithm_fill_n                                      [system] { header "__algorithm/fill_n.h" }
-module std_private_algorithm_find                                        [system] {
-  header "__algorithm/find.h"
-  export std_private_algorithm_unwrap_iter
-}
-module std_private_algorithm_find_end                                    [system] { header "__algorithm/find_end.h" }
-module std_private_algorithm_find_first_of                               [system] { header "__algorithm/find_first_of.h" }
-module std_private_algorithm_find_if                                     [system] { header "__algorithm/find_if.h" }
-module std_private_algorithm_find_if_not                                 [system] { header "__algorithm/find_if_not.h" }
-module std_private_algorithm_find_segment_if                             [system] { header "__algorithm/find_segment_if.h" }
-module std_private_algorithm_fold                                        [system] { header "__algorithm/fold.h" }
-module std_private_algorithm_for_each                                    [system] { header "__algorithm/for_each.h" }
-module std_private_algorithm_for_each_n                                  [system] { header "__algorithm/for_each_n.h" }
-module std_private_algorithm_for_each_segment                            [system] { header "__algorithm/for_each_segment.h" }
-module std_private_algorithm_generate                                    [system] { header "__algorithm/generate.h" }
-module std_private_algorithm_generate_n                                  [system] { header "__algorithm/generate_n.h" }
-module std_private_algorithm_half_positive                               [system] { header "__algorithm/half_positive.h" }
-module std_private_algorithm_in_found_result                             [system] { header "__algorithm/in_found_result.h" }
-module std_private_algorithm_in_fun_result                               [system] { header "__algorithm/in_fun_result.h" }
-module std_private_algorithm_in_in_out_result                            [system] { header "__algorithm/in_in_out_result.h" }
-module std_private_algorithm_in_in_result                                [system] { header "__algorithm/in_in_result.h" }
-module std_private_algorithm_in_out_out_result                           [system] { header "__algorithm/in_out_out_result.h" }
-module std_private_algorithm_in_out_result                               [system] { header "__algorithm/in_out_result.h" }
-module std_private_algorithm_includes                                    [system] { header "__algorithm/includes.h" }
-module std_private_algorithm_inplace_merge                               [system] { header "__algorithm/inplace_merge.h" }
-module std_private_algorithm_is_heap                                     [system] { header "__algorithm/is_heap.h" }
-module std_private_algorithm_is_heap_until                               [system] { header "__algorithm/is_heap_until.h" }
-module std_private_algorithm_is_partitioned                              [system] { header "__algorithm/is_partitioned.h" }
-module std_private_algorithm_is_permutation                              [system] { header "__algorithm/is_permutation.h" }
-module std_private_algorithm_is_sorted                                   [system] { header "__algorithm/is_sorted.h" }
-module std_private_algorithm_is_sorted_until                             [system] { header "__algorithm/is_sorted_until.h" }
-module std_private_algorithm_iter_swap                                   [system] { header "__algorithm/iter_swap.h" }
-module std_private_algorithm_iterator_operations                         [system] {
-  header "__algorithm/iterator_operations.h"
-  export *
-}
-module std_private_algorithm_lexicographical_compare                     [system] { header "__algorithm/lexicographical_compare.h" }
-module std_private_algorithm_lexicographical_compare_three_way           [system] { header "__algorithm/lexicographical_compare_three_way.h" }
-module std_private_algorithm_lower_bound                                 [system] { header "__algorithm/lower_bound.h" }
-module std_private_algorithm_make_heap                                   [system] { header "__algorithm/make_heap.h" }
-module std_private_algorithm_make_projected                              [system] { header "__algorithm/make_projected.h" }
-module std_private_algorithm_max                                         [system] { header "__algorithm/max.h" }
-module std_private_algorithm_max_element                                 [system] { header "__algorithm/max_element.h" }
-module std_private_algorithm_merge                                       [system] { header "__algorithm/merge.h" }
-module std_private_algorithm_min                                         [system] { header "__algorithm/min.h" }
-module std_private_algorithm_min_element                                 [system] { header "__algorithm/min_element.h" }
-module std_private_algorithm_min_max_result                              [system] { header "__algorithm/min_max_result.h" }
-module std_private_algorithm_minmax                                      [system] {
-  header "__algorithm/minmax.h"
-  export *
-}
-module std_private_algorithm_minmax_element                              [system] { header "__algorithm/minmax_element.h" }
-module std_private_algorithm_mismatch                                    [system] {
-  header "__algorithm/mismatch.h"
-  export std_private_algorithm_simd_utils
-  export std_private_iterator_aliasing_iterator
-}
-module std_private_algorithm_move                                        [system] { header "__algorithm/move.h" }
-module std_private_algorithm_move_backward                               [system] { header "__algorithm/move_backward.h" }
-module std_private_algorithm_next_permutation                            [system] { header "__algorithm/next_permutation.h" }
-module std_private_algorithm_none_of                                     [system] { header "__algorithm/none_of.h" }
-module std_private_algorithm_nth_element                                 [system] { header "__algorithm/nth_element.h" }
-module std_private_algorithm_partial_sort                                [system] { header "__algorithm/partial_sort.h" }
-module std_private_algorithm_partial_sort_copy                           [system] { header "__algorithm/partial_sort_copy.h" }
-module std_private_algorithm_partition                                   [system] { header "__algorithm/partition.h" }
-module std_private_algorithm_partition_copy                              [system] { header "__algorithm/partition_copy.h" }
-module std_private_algorithm_partition_point                             [system] { header "__algorithm/partition_point.h" }
-module std_private_algorithm_pop_heap                                    [system] { header "__algorithm/pop_heap.h" }
-module std_private_algorithm_prev_permutation                            [system] { header "__algorithm/prev_permutation.h" }
-module std_private_algorithm_pstl                                        [system] {
-  header "__algorithm/pstl.h"
-  export *
-}
-module std_private_algorithm_push_heap                                   [system] { header "__algorithm/push_heap.h" }
-module std_private_algorithm_ranges_adjacent_find                        [system] { header "__algorithm/ranges_adjacent_find.h" }
-module std_private_algorithm_ranges_all_of                               [system] { header "__algorithm/ranges_all_of.h" }
-module std_private_algorithm_ranges_any_of                               [system] { header "__algorithm/ranges_any_of.h" }
-module std_private_algorithm_ranges_binary_search                        [system] {
-  header "__algorithm/ranges_binary_search.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_clamp                                [system] {
-  header "__algorithm/ranges_clamp.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_contains                             [system] { header "__algorithm/ranges_contains.h" }
-module std_private_algorithm_ranges_contains_subrange                    [system] { header "__algorithm/ranges_contains_subrange.h" }
-module std_private_algorithm_ranges_copy                                 [system] {
-  header "__algorithm/ranges_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_copy_backward                        [system] {
-  header "__algorithm/ranges_copy_backward.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_copy_if                              [system] {
-  header "__algorithm/ranges_copy_if.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_copy_n                               [system] {
-  header "__algorithm/ranges_copy_n.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_count                                [system] { header "__algorithm/ranges_count.h" }
-module std_private_algorithm_ranges_count_if                             [system] { header "__algorithm/ranges_count_if.h" }
-module std_private_algorithm_ranges_ends_with                            [system] { header "__algorithm/ranges_ends_with.h" }
-module std_private_algorithm_ranges_equal                                [system] { header "__algorithm/ranges_equal.h" }
-module std_private_algorithm_ranges_equal_range                          [system] {
-  header "__algorithm/ranges_equal_range.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_fill                                 [system] { header "__algorithm/ranges_fill.h" }
-module std_private_algorithm_ranges_fill_n                               [system] { header "__algorithm/ranges_fill_n.h" }
-module std_private_algorithm_ranges_find                                 [system] { header "__algorithm/ranges_find.h" }
-module std_private_algorithm_ranges_find_end                             [system] { header "__algorithm/ranges_find_end.h" }
-module std_private_algorithm_ranges_find_first_of                        [system] { header "__algorithm/ranges_find_first_of.h" }
-module std_private_algorithm_ranges_find_if                              [system] { header "__algorithm/ranges_find_if.h" }
-module std_private_algorithm_ranges_find_if_not                          [system] { header "__algorithm/ranges_find_if_not.h" }
-module std_private_algorithm_ranges_find_last                            [system] { header "__algorithm/ranges_find_last.h" }
-module std_private_algorithm_ranges_for_each                             [system] {
-  header "__algorithm/ranges_for_each.h"
-  export std_private_algorithm_in_fun_result
-}
-module std_private_algorithm_ranges_for_each_n                           [system] {
-  header "__algorithm/ranges_for_each_n.h"
-  export std_private_algorithm_in_fun_result
-}
-module std_private_algorithm_ranges_generate                             [system] { header "__algorithm/ranges_generate.h" }
-module std_private_algorithm_ranges_generate_n                           [system] { header "__algorithm/ranges_generate_n.h" }
-module std_private_algorithm_ranges_includes                             [system] {
-  header "__algorithm/ranges_includes.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_inplace_merge                        [system] {
-  header "__algorithm/ranges_inplace_merge.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_is_heap                              [system] {
-  header "__algorithm/ranges_is_heap.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_is_heap_until                        [system] {
-  header "__algorithm/ranges_is_heap_until.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_is_partitioned                       [system] { header "__algorithm/ranges_is_partitioned.h" }
-module std_private_algorithm_ranges_is_permutation                       [system] { header "__algorithm/ranges_is_permutation.h" }
-module std_private_algorithm_ranges_is_sorted                            [system] {
-  header "__algorithm/ranges_is_sorted.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_is_sorted_until                      [system] {
-  header "__algorithm/ranges_is_sorted_until.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_iterator_concept                     [system] { header "__algorithm/ranges_iterator_concept.h" }
-module std_private_algorithm_ranges_lexicographical_compare              [system] {
-  header "__algorithm/ranges_lexicographical_compare.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_lower_bound                          [system] {
-  header "__algorithm/ranges_lower_bound.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_make_heap                            [system] {
-  header "__algorithm/ranges_make_heap.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_max                                  [system] {
-  header "__algorithm/ranges_max.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_max_element                          [system] {
-  header "__algorithm/ranges_max_element.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_merge                                [system] {
-  header "__algorithm/ranges_merge.h"
-  export std_private_algorithm_in_in_out_result
-}
-module std_private_algorithm_ranges_min                                  [system] {
-  header "__algorithm/ranges_min.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_min_element                          [system] {
-  header "__algorithm/ranges_min_element.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_minmax                               [system] {
-  header "__algorithm/ranges_minmax.h"
-  export std_private_functional_ranges_operations
-  export std_private_algorithm_min_max_result
-}
-module std_private_algorithm_ranges_minmax_element                       [system] {
-  header "__algorithm/ranges_minmax_element.h"
-  export std_private_functional_ranges_operations
-  export std_private_algorithm_min_max_result
-}
-module std_private_algorithm_ranges_mismatch                             [system] {
-  header "__algorithm/ranges_mismatch.h"
-  export std_private_algorithm_in_in_result
-}
-module std_private_algorithm_ranges_move                                 [system] {
-  header "__algorithm/ranges_move.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_move_backward                        [system] {
-  header "__algorithm/ranges_move_backward.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_next_permutation                     [system] {
-  header "__algorithm/ranges_next_permutation.h"
-  export std_private_algorithm_in_found_result
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_none_of                              [system] { header "__algorithm/ranges_none_of.h" }
-module std_private_algorithm_ranges_nth_element                          [system] {
-  header "__algorithm/ranges_nth_element.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_partial_sort                         [system] {
-  header "__algorithm/ranges_partial_sort.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_partial_sort_copy                    [system] {
-  header "__algorithm/ranges_partial_sort_copy.h"
-  export std_private_algorithm_in_out_result
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_partition                            [system] { header "__algorithm/ranges_partition.h" }
-module std_private_algorithm_ranges_partition_copy                       [system] { header "__algorithm/ranges_partition_copy.h" }
-module std_private_algorithm_ranges_partition_point                      [system] { header "__algorithm/ranges_partition_point.h" }
-module std_private_algorithm_ranges_pop_heap                             [system] {
-  header "__algorithm/ranges_pop_heap.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_prev_permutation                     [system] {
-  header "__algorithm/ranges_prev_permutation.h"
-  export std_private_algorithm_in_found_result
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_push_heap                            [system] {
-  header "__algorithm/ranges_push_heap.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_remove                               [system] { header "__algorithm/ranges_remove.h" }
-module std_private_algorithm_ranges_remove_copy                          [system] {
-  header "__algorithm/ranges_remove_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_remove_copy_if                       [system] {
-  header "__algorithm/ranges_remove_copy_if.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_remove_if                            [system] { header "__algorithm/ranges_remove_if.h" }
-module std_private_algorithm_ranges_replace                              [system] { header "__algorithm/ranges_replace.h" }
-module std_private_algorithm_ranges_replace_copy                         [system] {
-  header "__algorithm/ranges_replace_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_replace_copy_if                      [system] {
-  header "__algorithm/ranges_replace_copy_if.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_replace_if                           [system] { header "__algorithm/ranges_replace_if.h" }
-module std_private_algorithm_ranges_reverse                              [system] { header "__algorithm/ranges_reverse.h" }
-module std_private_algorithm_ranges_reverse_copy                         [system] {
-  header "__algorithm/ranges_reverse_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_rotate                               [system] { header "__algorithm/ranges_rotate.h" }
-module std_private_algorithm_ranges_rotate_copy                          [system] {
-  header "__algorithm/ranges_rotate_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_sample                               [system] { header "__algorithm/ranges_sample.h" }
-module std_private_algorithm_ranges_search                               [system] { header "__algorithm/ranges_search.h" }
-module std_private_algorithm_ranges_search_n                             [system] { header "__algorithm/ranges_search_n.h" }
-module std_private_algorithm_ranges_set_difference                       [system] {
-  header "__algorithm/ranges_set_difference.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_set_intersection                     [system] {
-  header "__algorithm/ranges_set_intersection.h"
-  export std_private_algorithm_in_in_out_result
-}
-module std_private_algorithm_ranges_set_symmetric_difference             [system] {
-  header "__algorithm/ranges_set_symmetric_difference.h"
-  export std_private_algorithm_in_in_out_result
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_set_union                            [system] {
-  header "__algorithm/ranges_set_union.h"
-  export std_private_algorithm_in_in_out_result
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_shuffle                              [system] { header "__algorithm/ranges_shuffle.h" }
-module std_private_algorithm_ranges_sort                                 [system] {
-  header "__algorithm/ranges_sort.h"
-  export std_private_algorithm_make_projected
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_sort_heap                            [system] {
-  header "__algorithm/ranges_sort_heap.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_stable_partition                     [system] { header "__algorithm/ranges_stable_partition.h" }
-module std_private_algorithm_ranges_stable_sort                          [system] {
-  header "__algorithm/ranges_stable_sort.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_ranges_starts_with                          [system] { header "__algorithm/ranges_starts_with.h" }
-module std_private_algorithm_ranges_swap_ranges                          [system] {
-  header "__algorithm/ranges_swap_ranges.h"
-  export std_private_algorithm_in_in_result
-}
-module std_private_algorithm_ranges_transform                            [system] {
-  header "__algorithm/ranges_transform.h"
-  export std_private_algorithm_in_in_out_result
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_unique                               [system] { header "__algorithm/ranges_unique.h" }
-module std_private_algorithm_ranges_unique_copy                          [system] {
-  header "__algorithm/ranges_unique_copy.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_algorithm_ranges_upper_bound                          [system] {
-  header "__algorithm/ranges_upper_bound.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_algorithm_remove                                      [system] { header "__algorithm/remove.h" }
-module std_private_algorithm_remove_copy                                 [system] { header "__algorithm/remove_copy.h" }
-module std_private_algorithm_remove_copy_if                              [system] { header "__algorithm/remove_copy_if.h" }
-module std_private_algorithm_remove_if                                   [system] { header "__algorithm/remove_if.h" }
-module std_private_algorithm_replace                                     [system] { header "__algorithm/replace.h" }
-module std_private_algorithm_replace_copy                                [system] { header "__algorithm/replace_copy.h" }
-module std_private_algorithm_replace_copy_if                             [system] { header "__algorithm/replace_copy_if.h" }
-module std_private_algorithm_replace_if                                  [system] { header "__algorithm/replace_if.h" }
-module std_private_algorithm_reverse                                     [system] { header "__algorithm/reverse.h" }
-module std_private_algorithm_reverse_copy                                [system] { header "__algorithm/reverse_copy.h" }
-module std_private_algorithm_rotate                                      [system] { header "__algorithm/rotate.h" }
-module std_private_algorithm_rotate_copy                                 [system] { header "__algorithm/rotate_copy.h" }
-module std_private_algorithm_sample                                      [system] { header "__algorithm/sample.h" }
-module std_private_algorithm_search                                      [system] { header "__algorithm/search.h" }
-module std_private_algorithm_search_n                                    [system] { header "__algorithm/search_n.h" }
-module std_private_algorithm_set_difference                              [system] { header "__algorithm/set_difference.h" }
-module std_private_algorithm_set_intersection                            [system] { header "__algorithm/set_intersection.h" }
-module std_private_algorithm_set_symmetric_difference                    [system] { header "__algorithm/set_symmetric_difference.h" }
-module std_private_algorithm_set_union                                   [system] { header "__algorithm/set_union.h" }
-module std_private_algorithm_shift_left                                  [system] { header "__algorithm/shift_left.h" }
-module std_private_algorithm_shift_right                                 [system] { header "__algorithm/shift_right.h" }
-module std_private_algorithm_shuffle                                     [system] { header "__algorithm/shuffle.h" }
-module std_private_algorithm_sift_down                                   [system] { header "__algorithm/sift_down.h" }
-module std_private_algorithm_sort                                        [system] {
-  header "__algorithm/sort.h"
-  export std_private_debug_utils_strict_weak_ordering_check
-}
-module std_private_algorithm_simd_utils                                  [system] { header "__algorithm/simd_utils.h" }
-module std_private_algorithm_sort_heap                                   [system] { header "__algorithm/sort_heap.h" }
-module std_private_algorithm_stable_partition                            [system] { header "__algorithm/stable_partition.h" }
-module std_private_algorithm_stable_sort                                 [system] { header "__algorithm/stable_sort.h" }
-module std_private_algorithm_swap_ranges                                 [system] {
-  header "__algorithm/swap_ranges.h"
-  export std_private_algorithm_iterator_operations
-}
-module std_private_algorithm_three_way_comp_ref_type                     [system] { header "__algorithm/three_way_comp_ref_type.h" }
-module std_private_algorithm_transform                                   [system] { header "__algorithm/transform.h" }
-module std_private_algorithm_uniform_random_bit_generator_adaptor        [system] { header "__algorithm/uniform_random_bit_generator_adaptor.h" }
-module std_private_algorithm_unique                                      [system] { header "__algorithm/unique.h" }
-module std_private_algorithm_unique_copy                                 [system] { header "__algorithm/unique_copy.h" }
-module std_private_algorithm_unwrap_iter                                 [system] {
-  header "__algorithm/unwrap_iter.h"
-  export std_private_iterator_iterator_traits
-}
-module std_private_algorithm_unwrap_range                                [system] {
-  header "__algorithm/unwrap_range.h"
-  export std_private_utility_pair
-}
-module std_private_algorithm_upper_bound                                 [system] { header "__algorithm/upper_bound.h" }
+  module variant {
+    module fwd       { header "__fwd/variant.h" }
+    module monostate { header "__variant/monostate.h" }
 
-module std_private_array_array_fwd [system] { header "__fwd/array.h" }
+    header "variant"
+    export *
+  }
 
-module std_private_atomic_aliases             [system] {
-  header "__atomic/aliases.h"
-  export std_private_atomic_atomic
-}
-module std_private_atomic_atomic              [system] {
-  header "__atomic/atomic.h"
-  export std_private_atomic_atomic_base
-}
-module std_private_atomic_atomic_base         [system] { header "__atomic/atomic_base.h" }
-module std_private_atomic_atomic_flag         [system] {
-  header "__atomic/atomic_flag.h"
-  export *
-}
-module std_private_atomic_atomic_init         [system] { header "__atomic/atomic_init.h" }
-module std_private_atomic_atomic_lock_free    [system] { header "__atomic/atomic_lock_free.h" }
-module std_private_atomic_atomic_ref          [system] { header "__atomic/atomic_ref.h" }
-module std_private_atomic_atomic_sync         [system] {
-  header "__atomic/atomic_sync.h"
-  export std_private_atomic_to_gcc_order
-}
-module std_private_atomic_check_memory_order  [system] { header "__atomic/check_memory_order.h" }
-module std_private_atomic_contention_t        [system] { header "__atomic/contention_t.h" }
-module std_private_atomic_cxx_atomic_impl     [system] { header "__atomic/cxx_atomic_impl.h" }
-module std_private_atomic_fence               [system] { header "__atomic/fence.h" }
-module std_private_atomic_is_always_lock_free [system] { header "__atomic/is_always_lock_free.h" }
-module std_private_atomic_kill_dependency     [system] { header "__atomic/kill_dependency.h" }
-module std_private_atomic_memory_order        [system] { header "__atomic/memory_order.h" }
-module std_private_atomic_to_gcc_order        [system] {
-  header "__atomic/to_gcc_order.h"
-  export std_private_atomic_memory_order
-}
+  module vector {
+    module fwd { header "__fwd/vector.h" }
 
-module std_private_bit_bit_cast       [system] { header "__bit/bit_cast.h" }
-module std_private_bit_bit_ceil       [system] { header "__bit/bit_ceil.h" }
-module std_private_bit_bit_floor      [system] { header "__bit/bit_floor.h" }
-module std_private_bit_bit_log2       [system] { header "__bit/bit_log2.h" }
-module std_private_bit_bit_width      [system] { header "__bit/bit_width.h" }
-module std_private_bit_blsr           [system] { header "__bit/blsr.h" }
-module std_private_bit_byteswap       [system] { header "__bit/byteswap.h" }
-module std_private_bit_countl         [system] { header "__bit/countl.h" }
-module std_private_bit_countr         [system] { header "__bit/countr.h" }
-module std_private_bit_endian         [system] { header "__bit/endian.h" }
-module std_private_bit_has_single_bit [system] { header "__bit/has_single_bit.h" }
-module std_private_bit_invert_if      [system] { header "__bit/invert_if.h" }
-module std_private_bit_popcount       [system] { header "__bit/popcount.h" }
-module std_private_bit_rotate         [system] { header "__bit/rotate.h" }
-
-module std_private_chrono_calendar               [system] { header "__chrono/calendar.h" }
-module std_private_chrono_concepts               [system] { header "__chrono/concepts.h" }
-module std_private_chrono_convert_to_timespec    [system] { header "__chrono/convert_to_timespec.h" }
-module std_private_chrono_convert_to_tm          [system] { header "__chrono/convert_to_tm.h" }
-module std_private_chrono_day                    [system] { header "__chrono/day.h" }
-module std_private_chrono_duration               [system] {
-  header "__chrono/duration.h"
-  export std_private_type_traits_is_convertible
-}
-module std_private_chrono_exception              [system] { header "__chrono/exception.h" }
-module std_private_chrono_file_clock             [system] { header "__chrono/file_clock.h" }
-module std_private_chrono_formatter              [system] {
-  header "__chrono/formatter.h"
-}
-module std_private_chrono_hh_mm_ss               [system] { header "__chrono/hh_mm_ss.h" }
-module std_private_chrono_high_resolution_clock  [system] {
-  header "__chrono/high_resolution_clock.h"
-  export std_private_chrono_steady_clock
-  export std_private_chrono_system_clock
-}
-module std_private_chrono_leap_second            [system] { header "__chrono/leap_second.h" }
-module std_private_chrono_literals               [system] { header "__chrono/literals.h" }
-module std_private_chrono_local_info             [system] {
-  header "__chrono/local_info.h"
-  export std_private_chrono_sys_info
-}
-module std_private_chrono_month                  [system] { header "__chrono/month.h" }
-module std_private_chrono_month_weekday          [system] { header "__chrono/month_weekday.h" }
-module std_private_chrono_monthday               [system] { header "__chrono/monthday.h" }
-module std_private_chrono_ostream                [system] {
-  header "__chrono/ostream.h"
-}
-module std_private_chrono_parser_std_format_spec [system] {
-  header "__chrono/parser_std_format_spec.h"
-}
-module std_private_chrono_statically_widen       [system] { header "__chrono/statically_widen.h" }
-module std_private_chrono_steady_clock           [system] {
-  header "__chrono/steady_clock.h"
-  export std_private_chrono_time_point
-}
-module std_private_chrono_time_zone              [system] {
-  header "__chrono/time_zone.h"
-  export std_private_memory_unique_ptr
-}
-module std_private_chrono_time_zone_link         [system] {
-  header "__chrono/time_zone_link.h"
-}
-module std_private_chrono_sys_info               [system] {
-  header "__chrono/sys_info.h"
-}
-module std_private_chrono_system_clock           [system] {
-  header "__chrono/system_clock.h"
-  export std_private_chrono_time_point
-}
-module std_private_chrono_tzdb                   [system] {
-  header "__chrono/tzdb.h"
-  export *
-}
-module std_private_chrono_tzdb_list              [system] {
-  header "__chrono/tzdb_list.h"
-  export *
-}
-module std_private_chrono_time_point             [system] { header "__chrono/time_point.h" }
-module std_private_chrono_weekday                [system] { header "__chrono/weekday.h" }
-module std_private_chrono_year                   [system] { header "__chrono/year.h" }
-module std_private_chrono_year_month             [system] { header "__chrono/year_month.h" }
-module std_private_chrono_year_month_day         [system] { header "__chrono/year_month_day.h" }
-module std_private_chrono_year_month_weekday     [system] { header "__chrono/year_month_weekday.h" }
-module std_private_chrono_zoned_time             [system] { header "__chrono/zoned_time.h" }
-
-module std_private_compare_common_comparison_category     [system] { header "__compare/common_comparison_category.h" }
-module std_private_compare_compare_partial_order_fallback [system] { header "__compare/compare_partial_order_fallback.h" }
-module std_private_compare_compare_strong_order_fallback  [system] { header "__compare/compare_strong_order_fallback.h" }
-module std_private_compare_compare_three_way              [system] { header "__compare/compare_three_way.h" }
-module std_private_compare_compare_three_way_result       [system] { header "__compare/compare_three_way_result.h" }
-module std_private_compare_compare_weak_order_fallback    [system] { header "__compare/compare_weak_order_fallback.h" }
-module std_private_compare_is_eq                          [system] { header "__compare/is_eq.h" }
-module std_private_compare_ordering                       [system] { header "__compare/ordering.h" }
-module std_private_compare_partial_order                  [system] { header "__compare/partial_order.h" }
-module std_private_compare_strong_order                   [system] { header "__compare/strong_order.h" }
-module std_private_compare_synth_three_way                [system] { header "__compare/synth_three_way.h" }
-module std_private_compare_three_way_comparable           [system] { header "__compare/three_way_comparable.h" }
-module std_private_compare_weak_order                     [system] { header "__compare/weak_order.h" }
-
-module std_private_complex_complex_fwd            [system] { header "__fwd/complex.h" }
-
-module std_private_concepts_arithmetic            [system] { header "__concepts/arithmetic.h" }
-module std_private_concepts_assignable            [system] { header "__concepts/assignable.h" }
-module std_private_concepts_boolean_testable      [system] { header "__concepts/boolean_testable.h" }
-module std_private_concepts_class_or_enum         [system] { header "__concepts/class_or_enum.h" }
-module std_private_concepts_common_reference_with [system] { header "__concepts/common_reference_with.h" }
-module std_private_concepts_common_with           [system] { header "__concepts/common_with.h" }
-module std_private_concepts_constructible         [system] {
-  header "__concepts/constructible.h"
-  export std_private_concepts_destructible
-}
-module std_private_concepts_convertible_to        [system] { header "__concepts/convertible_to.h" }
-module std_private_concepts_copyable              [system] { header "__concepts/copyable.h" }
-module std_private_concepts_derived_from          [system] { header "__concepts/derived_from.h" }
-module std_private_concepts_destructible          [system] {
-  header "__concepts/destructible.h"
-  export std_private_type_traits_is_nothrow_destructible
-}
-module std_private_concepts_different_from        [system] { header "__concepts/different_from.h" }
-module std_private_concepts_equality_comparable   [system] {
-  header "__concepts/equality_comparable.h"
-  export std_private_type_traits_common_reference
-}
-module std_private_concepts_invocable             [system] { header "__concepts/invocable.h" }
-module std_private_concepts_movable               [system] {
-  header "__concepts/movable.h"
-  export std_private_type_traits_is_object
-}
-module std_private_concepts_predicate             [system] { header "__concepts/predicate.h" }
-module std_private_concepts_regular               [system] { header "__concepts/regular.h" }
-module std_private_concepts_relation              [system] { header "__concepts/relation.h" }
-module std_private_concepts_same_as               [system] {
-  header "__concepts/same_as.h"
-  export std_private_type_traits_is_same
-}
-module std_private_concepts_semiregular           [system] { header "__concepts/semiregular.h" }
-module std_private_concepts_swappable             [system] { header "__concepts/swappable.h" }
-module std_private_concepts_totally_ordered       [system] { header "__concepts/totally_ordered.h" }
-
-module std_private_debug_utils_randomize_range            [system] { header "__debug_utils/randomize_range.h" }
-module std_private_debug_utils_sanitizers                 [system] { header "__debug_utils/sanitizers.h" }
-module std_private_debug_utils_strict_weak_ordering_check [system] {
-  header "__debug_utils/strict_weak_ordering_check.h"
-  export std_private_type_traits_is_constant_evaluated
-}
+    header "vector"
+    export *
+  }
 
-module std_private_deque_fwd [system] { header "__fwd/deque.h" }
+  // Experimental C++ Standard Library interfaces
+  module experimental {
+    module iterator           { header "experimental/iterator" }
+    module memory             { header "experimental/memory" }
+    module propagate_const    { header "experimental/propagate_const" }
+    module type_traits        { header "experimental/type_traits" }
+    module utility            { header "experimental/utility" }
+    module simd {
+      private header "experimental/__simd/aligned_tag.h"
+      private header "experimental/__simd/declaration.h"
+      private header "experimental/__simd/reference.h"
+      private header "experimental/__simd/scalar.h"
+      private header "experimental/__simd/simd_mask.h"
+      private header "experimental/__simd/simd.h"
+      private header "experimental/__simd/traits.h"
+      private header "experimental/__simd/utility.h"
+      private header "experimental/__simd/vec_ext.h"
+      header "experimental/simd"
+      export *
+    }
+  }
 
-module std_private_exception_exception        [system] { header "__exception/exception.h" }
-module std_private_exception_exception_ptr    [system] {
-  header "__exception/exception_ptr.h"
-  export std_private_exception_operations
-}
-module std_private_exception_nested_exception [system] { header "__exception/nested_exception.h" }
-module std_private_exception_operations       [system] { header "__exception/operations.h" }
-module std_private_exception_terminate        [system] { header "__exception/terminate.h" }
-
-module std_private_expected_bad_expected_access [system] { header "__expected/bad_expected_access.h" }
-module std_private_expected_expected            [system] { header "__expected/expected.h" }
-module std_private_expected_unexpect            [system] { header "__expected/unexpect.h" }
-module std_private_expected_unexpected          [system] { header "__expected/unexpected.h" }
-
-module std_private_format_buffer                          [system] { header "__format/buffer.h" }
-module std_private_format_concepts                        [system] { header "__format/concepts.h" }
-module std_private_format_container_adaptor               [system] { header "__format/container_adaptor.h" }
-module std_private_format_enable_insertable               [system] { header "__format/enable_insertable.h" }
-module std_private_format_escaped_output_table            [system] { header "__format/escaped_output_table.h" }
-module std_private_format_extended_grapheme_cluster_table [system] { header "__format/extended_grapheme_cluster_table.h" }
-module std_private_format_format_arg                      [system] { header "__format/format_arg.h" }
-module std_private_format_format_arg_store                [system] { header "__format/format_arg_store.h" }
-module std_private_format_format_args                     [system] { header "__format/format_args.h" }
-module std_private_format_format_context                  [system] {
-  header "__format/format_context.h"
-  export *
-}
-module std_private_format_format_error                    [system] { header "__format/format_error.h" }
-module std_private_format_format_functions                [system] {
-  header "__format/format_functions.h"
-  export std_string
-}
-module std_private_format_fwd                             [system] { header "__fwd/format.h" }
-module std_private_format_format_parse_context            [system] { header "__format/format_parse_context.h" }
-module std_private_format_format_string                   [system] { header "__format/format_string.h" }
-module std_private_format_format_to_n_result              [system] {
-  header "__format/format_to_n_result.h"
-  export std_private_iterator_incrementable_traits
-}
-module std_private_format_formatter                       [system] { header "__format/formatter.h" }
-module std_private_format_formatter_bool                  [system] { header "__format/formatter_bool.h" }
-module std_private_format_formatter_char                  [system] { header "__format/formatter_char.h" }
-module std_private_format_formatter_floating_point        [system] { header "__format/formatter_floating_point.h" }
-module std_private_format_formatter_integer               [system] { header "__format/formatter_integer.h" }
-module std_private_format_formatter_integral              [system] { header "__format/formatter_integral.h" }
-module std_private_format_formatter_output                [system] { header "__format/formatter_output.h" }
-module std_private_format_formatter_pointer               [system] { header "__format/formatter_pointer.h" }
-module std_private_format_formatter_string                [system] { header "__format/formatter_string.h" }
-module std_private_format_formatter_tuple                 [system] { header "__format/formatter_tuple.h" }
-module std_private_format_indic_conjunct_break_table      [system] { header "__format/indic_conjunct_break_table.h" }
-module std_private_format_parser_std_format_spec          [system] { header "__format/parser_std_format_spec.h" }
-module std_private_format_range_default_formatter         [system] { header "__format/range_default_formatter.h" }
-module std_private_format_range_formatter                 [system] { header "__format/range_formatter.h" }
-module std_private_format_unicode                         [system] {
-  header "__format/unicode.h"
-  export std_private_format_extended_grapheme_cluster_table
-  export std_private_format_indic_conjunct_break_table
-}
-module std_private_format_width_estimation_table          [system] { header "__format/width_estimation_table.h" }
-module std_private_format_write_escaped                   [system] { header "__format/write_escaped.h" }
-
-module std_private_functional_binary_function            [system] { header "__functional/binary_function.h" }
-module std_private_functional_binary_negate              [system] { header "__functional/binary_negate.h" }
-module std_private_functional_bind                       [system] { header "__functional/bind.h" }
-module std_private_functional_bind_back                  [system] { header "__functional/bind_back.h" }
-module std_private_functional_bind_front                 [system] { header "__functional/bind_front.h" }
-module std_private_functional_binder1st                  [system] { header "__functional/binder1st.h" }
-module std_private_functional_binder2nd                  [system] { header "__functional/binder2nd.h" }
-module std_private_functional_boyer_moore_searcher       [system] {
-  header "__functional/boyer_moore_searcher.h"
-  export std_private_memory_shared_ptr
-}
-module std_private_functional_compose                    [system] {
-  header "__functional/compose.h"
-  export std_private_functional_perfect_forward
-}
-module std_private_functional_default_searcher           [system] { header "__functional/default_searcher.h" }
-module std_private_functional_function                   [system] { header "__functional/function.h" }
-module std_private_functional_hash                       [system] {
-  header "__functional/hash.h"
-  export std_cstdint
-  export std_private_type_traits_underlying_type
-  export std_private_utility_pair
-}
-module std_private_functional_fwd                        [system] { header "__fwd/functional.h" }
-module std_private_functional_identity                   [system] { header "__functional/identity.h" }
-module std_private_functional_invoke                     [system] {
-  header "__functional/invoke.h"
-  export *
-}
-module std_private_functional_is_transparent             [system] { header "__functional/is_transparent.h" }
-module std_private_functional_mem_fn                     [system] { header "__functional/mem_fn.h" }
-module std_private_functional_mem_fun_ref                [system] { header "__functional/mem_fun_ref.h" }
-module std_private_functional_not_fn                     [system] {
-  header "__functional/not_fn.h"
-  export std_private_functional_perfect_forward
-}
-module std_private_functional_operations                 [system] { header "__functional/operations.h" }
-module std_private_functional_perfect_forward            [system] {
-  header "__functional/perfect_forward.h"
-  export *
-}
-module std_private_functional_pointer_to_binary_function [system] { header "__functional/pointer_to_binary_function.h" }
-module std_private_functional_pointer_to_unary_function  [system] { header "__functional/pointer_to_unary_function.h" }
-module std_private_functional_ranges_operations          [system] { header "__functional/ranges_operations.h" }
-module std_private_functional_reference_wrapper          [system] { header "__functional/reference_wrapper.h" }
-module std_private_functional_unary_function             [system] { header "__functional/unary_function.h" }
-module std_private_functional_unary_negate               [system] { header "__functional/unary_negate.h" }
-module std_private_functional_weak_result_type           [system] { header "__functional/weak_result_type.h" }
-
-module std_private_ios_fpos [system] { header "__ios/fpos.h" }
-
-module std_private_iosfwd_fstream_fwd   [system] { header "__fwd/fstream.h" }
-module std_private_iosfwd_ios_fwd       [system] { header "__fwd/ios.h" }
-module std_private_iosfwd_istream_fwd   [system] { header "__fwd/istream.h" }
-module std_private_iosfwd_ostream_fwd   [system] { header "__fwd/ostream.h" }
-module std_private_iosfwd_sstream_fwd   [system] { header "__fwd/sstream.h" }
-module std_private_iosfwd_streambuf_fwd [system] { header "__fwd/streambuf.h" }
-
-module std_private_iterator_access                  [system] { header "__iterator/access.h" }
-module std_private_iterator_advance                 [system] { header "__iterator/advance.h" }
-module std_private_iterator_aliasing_iterator       [system] { header "__iterator/aliasing_iterator.h" }
-module std_private_iterator_back_insert_iterator    [system] { header "__iterator/back_insert_iterator.h" }
-module std_private_iterator_bounded_iter            [system] { header "__iterator/bounded_iter.h" }
-module std_private_iterator_common_iterator         [system] { header "__iterator/common_iterator.h" }
-module std_private_iterator_concepts                [system] {
-  header "__iterator/concepts.h"
-  export std_private_concepts_constructible
-  export std_private_concepts_equality_comparable
-  export std_private_concepts_movable
-  export std_private_type_traits_common_reference
-  export std_private_type_traits_is_reference
-  export std_private_type_traits_remove_cvref
-}
-module std_private_iterator_counted_iterator        [system] { header "__iterator/counted_iterator.h" }
-module std_private_iterator_cpp17_iterator_concepts [system] { header "__iterator/cpp17_iterator_concepts.h" }
-module std_private_iterator_data                    [system] { header "__iterator/data.h" }
-module std_private_iterator_default_sentinel        [system] { header "__iterator/default_sentinel.h" }
-module std_private_iterator_distance                [system] {
-  header "__iterator/distance.h"
-  export std_private_ranges_size
-}
-module std_private_iterator_empty                   [system] { header "__iterator/empty.h" }
-module std_private_iterator_erase_if_container      [system] { header "__iterator/erase_if_container.h" }
-module std_private_iterator_front_insert_iterator   [system] { header "__iterator/front_insert_iterator.h" }
-module std_private_iterator_incrementable_traits    [system] { header "__iterator/incrementable_traits.h" }
-module std_private_iterator_indirectly_comparable   [system] { header "__iterator/indirectly_comparable.h" }
-module std_private_iterator_insert_iterator         [system] { header "__iterator/insert_iterator.h" }
-module std_private_iterator_istream_iterator        [system] { header "__iterator/istream_iterator.h" }
-module std_private_iterator_istreambuf_iterator     [system] { header "__iterator/istreambuf_iterator.h" }
-module std_private_iterator_iter_move               [system] { header "__iterator/iter_move.h" }
-module std_private_iterator_iter_swap               [system] { header "__iterator/iter_swap.h" }
-module std_private_iterator_iterator                [system] { header "__iterator/iterator.h" }
-module std_private_iterator_iterator_traits         [system] {
-  header "__iterator/iterator_traits.h"
-  export std_private_type_traits_is_primary_template
-  export std_private_type_traits_integral_constant
-}
-module std_private_iterator_iterator_with_data      [system] { header "__iterator/iterator_with_data.h" }
-module std_private_iterator_mergeable               [system] {
-  header "__iterator/mergeable.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_iterator_move_iterator           [system] { header "__iterator/move_iterator.h" }
-module std_private_iterator_move_sentinel           [system] { header "__iterator/move_sentinel.h" }
-module std_private_iterator_next                    [system] { header "__iterator/next.h" }
-module std_private_iterator_ostream_iterator        [system] { header "__iterator/ostream_iterator.h" }
-module std_private_iterator_ostreambuf_iterator     [system] {
-  header "__iterator/ostreambuf_iterator.h"
-  export *
-}
-module std_private_iterator_permutable              [system] { header "__iterator/permutable.h" }
-module std_private_iterator_prev                    [system] { header "__iterator/prev.h" }
-module std_private_iterator_projected               [system] { header "__iterator/projected.h" }
-module std_private_iterator_ranges_iterator_traits  [system] { header "__iterator/ranges_iterator_traits.h" }
-module std_private_iterator_readable_traits         [system] { header "__iterator/readable_traits.h" }
-module std_private_iterator_reverse_access          [system] { header "__iterator/reverse_access.h" }
-module std_private_iterator_reverse_iterator        [system] { header "__iterator/reverse_iterator.h" }
-module std_private_iterator_segmented_iterator      [system] { header "__iterator/segmented_iterator.h" }
-module std_private_iterator_size                    [system] { header "__iterator/size.h" }
-module std_private_iterator_sortable                [system] {
-  header "__iterator/sortable.h"
-  export std_private_functional_ranges_operations
-}
-module std_private_iterator_unreachable_sentinel    [system] { header "__iterator/unreachable_sentinel.h" }
-module std_private_iterator_wrap_iter               [system] { header "__iterator/wrap_iter.h" }
-
-module std_private_locale_locale_base_api_android              [system] { textual header "__locale_dir/locale_base_api/android.h" }
-module std_private_locale_locale_base_api_bsd_locale_defaults  [system] { textual header "__locale_dir/locale_base_api/bsd_locale_defaults.h" }
-module std_private_locale_locale_base_api_bsd_locale_fallbacks [system] { textual header "__locale_dir/locale_base_api/bsd_locale_fallbacks.h" }
-module std_private_locale_locale_base_api_fuchsia              [system] { textual header "__locale_dir/locale_base_api/fuchsia.h" }
-module std_private_locale_locale_base_api_ibm                  [system] { textual header "__locale_dir/locale_base_api/ibm.h" }
-module std_private_locale_locale_base_api_locale_guard         [system] { header "__locale_dir/locale_base_api/locale_guard.h" }
-module std_private_locale_locale_base_api_musl                 [system] { textual header "__locale_dir/locale_base_api/musl.h" }
-module std_private_locale_locale_base_api_newlib               [system] { textual header "__locale_dir/locale_base_api/newlib.h" }
-module std_private_locale_locale_base_api_openbsd              [system] { textual header "__locale_dir/locale_base_api/openbsd.h" }
-module std_private_locale_locale_base_api_win32                [system] { textual header "__locale_dir/locale_base_api/win32.h" }
-module std_private_locale_locale_base_api                      [system] {
-  header "__locale_dir/locale_base_api.h"
-  export *
-}
+  // Implementation detail headers that are private to libc++. These modules
+  // must not be directly imported.
+  module debug_utils {
+    module randomize_range              { header "__debug_utils/randomize_range.h" }
+    module sanitizers                   { header "__debug_utils/sanitizers.h" }
+    module strict_weak_ordering_check   { header "__debug_utils/strict_weak_ordering_check.h" }
+  }
 
-module std_private_math_abs                             [system] { header "__math/abs.h" }
-module std_private_math_copysign                        [system] { header "__math/copysign.h" }
-module std_private_math_error_functions                 [system] { header "__math/error_functions.h" }
-module std_private_math_exponential_functions           [system] { header "__math/exponential_functions.h" }
-module std_private_math_fdim                            [system] { header "__math/fdim.h" }
-module std_private_math_fma                             [system] { header "__math/fma.h" }
-module std_private_math_gamma                           [system] { header "__math/gamma.h" }
-module std_private_math_hyperbolic_functions            [system] { header "__math/hyperbolic_functions.h" }
-module std_private_math_hypot                           [system] { header "__math/hypot.h" }
-module std_private_math_inverse_hyperbolic_functions    [system] { header "__math/inverse_hyperbolic_functions.h" }
-module std_private_math_inverse_trigonometric_functions [system] { header "__math/inverse_trigonometric_functions.h" }
-module std_private_math_logarithms                      [system] { header "__math/logarithms.h" }
-module std_private_math_min_max                         [system] { header "__math/min_max.h" }
-module std_private_math_modulo                          [system] { header "__math/modulo.h" }
-module std_private_math_remainder                       [system] { header "__math/remainder.h" }
-module std_private_math_roots                           [system] { header "__math/roots.h" }
-module std_private_math_rounding_functions              [system] { header "__math/rounding_functions.h" }
-module std_private_math_special_functions               [system] { header "__math/special_functions.h" }
-module std_private_math_traits                          [system] { header "__math/traits.h" }
-module std_private_math_trigonometric_functions         [system] { header "__math/trigonometric_functions.h" }
-
-module std_private_memory_addressof                       [system] { header "__memory/addressof.h" }
-module std_private_memory_align                           [system] { header "__memory/align.h" }
-module std_private_memory_aligned_alloc                   [system] { header "__memory/aligned_alloc.h" }
-module std_private_memory_allocate_at_least               [system] { header "__memory/allocate_at_least.h" }
-module std_private_memory_allocation_guard                [system] { header "__memory/allocation_guard.h" }
-module std_private_memory_allocator                       [system] { header "__memory/allocator.h" }
-module std_private_memory_allocator_arg_t                 [system] { header "__memory/allocator_arg_t.h" }
-module std_private_memory_allocator_destructor            [system] { header "__memory/allocator_destructor.h" }
-module std_private_memory_allocator_traits                [system] { header "__memory/allocator_traits.h" }
-module std_private_memory_assume_aligned                  [system] { header "__memory/assume_aligned.h" }
-module std_private_memory_auto_ptr                        [system] { header "__memory/auto_ptr.h" }
-module std_private_memory_builtin_new_allocator           [system] {
-  header "__memory/builtin_new_allocator.h"
-  export *
-}
-module std_private_memory_compressed_pair                 [system] { header "__memory/compressed_pair.h" }
-module std_private_memory_concepts                        [system] {
-  header "__memory/concepts.h"
-  export std_private_type_traits_remove_reference
-}
-module std_private_memory_construct_at                    [system] { header "__memory/construct_at.h" }
-module std_private_memory_destruct_n                      [system] { header "__memory/destruct_n.h" }
-module std_private_memory_fwd                             [system] { header "__fwd/memory.h" }
-module std_private_memory_inout_ptr                       [system] { header "__memory/inout_ptr.h" }
-module std_private_memory_noexcept_move_assign_container  [system] { header "__memory/noexcept_move_assign_container.h" }
-module std_private_memory_out_ptr                         [system] { header "__memory/out_ptr.h" }
-module std_private_memory_pointer_traits                  [system] { header "__memory/pointer_traits.h" }
-module std_private_memory_ranges_construct_at             [system] { header "__memory/ranges_construct_at.h" }
-module std_private_memory_ranges_uninitialized_algorithms [system] {
-  header "__memory/ranges_uninitialized_algorithms.h"
-  export std_private_algorithm_in_out_result
-}
-module std_private_memory_raw_storage_iterator            [system] { header "__memory/raw_storage_iterator.h" }
-module std_private_memory_shared_ptr                      [system] {
-  header "__memory/shared_ptr.h"
-  export std_private_memory_uninitialized_algorithms
-}
-module std_private_memory_swap_allocator                  [system] { header "__memory/swap_allocator.h" }
-module std_private_memory_temp_value                      [system] { header "__memory/temp_value.h" }
-module std_private_memory_temporary_buffer                [system] {
-  header "__memory/temporary_buffer.h"
-  export std_private_utility_pair
-}
-module std_private_memory_uninitialized_algorithms        [system] {
-  header "__memory/uninitialized_algorithms.h"
-  export std_private_algorithm_copy
-}
-module std_private_memory_unique_ptr                      [system] {
-  header "__memory/unique_ptr.h"
-  export std_private_type_traits_add_lvalue_reference
-  export std_private_type_traits_is_pointer
-  export std_private_type_traits_type_identity
-}
-module std_private_memory_unique_temporary_buffer         [system] {
-  header "__memory/unique_temporary_buffer.h"
-  export std_private_memory_unique_ptr
-  export std_private_type_traits_is_constant_evaluated
-}
-module std_private_memory_uses_allocator                  [system] { header "__memory/uses_allocator.h" }
-module std_private_memory_uses_allocator_construction     [system] { header "__memory/uses_allocator_construction.h" }
-module std_private_memory_voidify                         [system] { header "__memory/voidify.h" }
-
-module std_private_memory_resource_memory_resource              [system] { header "__memory_resource/memory_resource.h" }
-module std_private_memory_resource_memory_resource_fwd          [system] { header "__fwd/memory_resource.h" }
-module std_private_memory_resource_monotonic_buffer_resource    [system] { header "__memory_resource/monotonic_buffer_resource.h" }
-module std_private_memory_resource_polymorphic_allocator        [system] { header "__memory_resource/polymorphic_allocator.h" }
-module std_private_memory_resource_pool_options                 [system] { header "__memory_resource/pool_options.h" }
-module std_private_memory_resource_synchronized_pool_resource   [system] {
-  header "__memory_resource/synchronized_pool_resource.h"
-  export *
-}
-module std_private_memory_resource_unsynchronized_pool_resource [system] { header "__memory_resource/unsynchronized_pool_resource.h" }
-
-module std_private_mutex_lock_guard  [system] { header "__mutex/lock_guard.h" }
-module std_private_mutex_mutex       [system] { header "__mutex/mutex.h" }
-module std_private_mutex_once_flag  [system]  { header "__mutex/once_flag.h" }
-module std_private_mutex_tag_types   [system] { header "__mutex/tag_types.h" }
-module std_private_mutex_unique_lock [system] { header "__mutex/unique_lock.h" }
-
-module std_private_numeric_accumulate               [system] { header "__numeric/accumulate.h" }
-module std_private_numeric_adjacent_difference      [system] { header "__numeric/adjacent_difference.h" }
-module std_private_numeric_exclusive_scan           [system] { header "__numeric/exclusive_scan.h" }
-module std_private_numeric_gcd_lcm                  [system] { header "__numeric/gcd_lcm.h" }
-module std_private_numeric_inclusive_scan           [system] { header "__numeric/inclusive_scan.h" }
-module std_private_numeric_inner_product            [system] { header "__numeric/inner_product.h" }
-module std_private_numeric_iota                     [system] { header "__numeric/iota.h" }
-module std_private_numeric_midpoint                 [system] { header "__numeric/midpoint.h" }
-module std_private_numeric_partial_sum              [system] { header "__numeric/partial_sum.h" }
-module std_private_numeric_pstl                     [system] {
-  header "__numeric/pstl.h"
-  export *
-}
-module std_private_numeric_reduce                   [system] { header "__numeric/reduce.h" }
-module std_private_numeric_saturation_arithmetic    [system] { header "__numeric/saturation_arithmetic.h" }
-module std_private_numeric_transform_exclusive_scan [system] { header "__numeric/transform_exclusive_scan.h" }
-module std_private_numeric_transform_inclusive_scan [system] { header "__numeric/transform_inclusive_scan.h" }
-module std_private_numeric_transform_reduce         [system] { header "__numeric/transform_reduce.h" }
-
-module std_private_pstl [system] {
-  header "__pstl/backend.h"
-  header "__pstl/backend_fwd.h"
-  header "__pstl/backends/default.h"
-  header "__pstl/backends/libdispatch.h"
-  header "__pstl/backends/serial.h"
-  header "__pstl/backends/std_thread.h"
-  header "__pstl/cpu_algos/any_of.h"
-  header "__pstl/cpu_algos/cpu_traits.h"
-  header "__pstl/cpu_algos/fill.h"
-  header "__pstl/cpu_algos/find_if.h"
-  header "__pstl/cpu_algos/for_each.h"
-  header "__pstl/cpu_algos/merge.h"
-  header "__pstl/cpu_algos/stable_sort.h"
-  header "__pstl/cpu_algos/transform.h"
-  header "__pstl/cpu_algos/transform_reduce.h"
-  header "__pstl/dispatch.h"
-  header "__pstl/handle_exception.h"
-}
+  module get_fwd {
+    header "__fwd/get.h"
+    export std_core.fwd.pair
+    export std_core.fwd.tuple
+    export std.array.fwd
+    export std.complex.fwd
+    export std.ranges.subrange_fwd
+    export std.variant.fwd
+  }
 
-module std_private_queue_fwd [system] { header "__fwd/queue.h" }
+  module pstl {
+    module backend_fwd {
+      header "__pstl/backend_fwd.h"
+    }
+    module backend {
+      header "__pstl/backend.h"
+      export * // need to export everything from whatever backend is currently configured
+    }
+    module backends {
+      module default {
+        header "__pstl/backends/default.h"
+        export std_core.utility_core.empty
+      }
+      module libdispatch {
+        header "__pstl/backends/libdispatch.h"
+        export std.pstl.cpu_algos
+        export std_core.utility_core.empty
+      }
+      module serial {
+        header "__pstl/backends/serial.h"
+        export std_core.utility_core.empty
+      }
+      module std_thread {
+        header "__pstl/backends/std_thread.h"
+        export std.pstl.cpu_algos
+        export std_core.utility_core.empty
+      }
+    }
+    module cpu_algos {
+      module any_of {
+        header "__pstl/cpu_algos/any_of.h"
+      }
+      module cpu_traits {
+        header "__pstl/cpu_algos/cpu_traits.h"
+      }
+      module fill {
+        header "__pstl/cpu_algos/fill.h"
+        export std_core.utility_core.empty
+      }
+      module find_if {
+        header "__pstl/cpu_algos/find_if.h"
+      }
+      module for_each {
+        header "__pstl/cpu_algos/for_each.h"
+        export std_core.utility_core.empty
+      }
+      module merge {
+        header "__pstl/cpu_algos/merge.h"
+      }
+      module stable_sort {
+        header "__pstl/cpu_algos/stable_sort.h"
+        export std_core.utility_core.empty
+      }
+      module transform {
+        header "__pstl/cpu_algos/transform.h"
+      }
+      module transform_reduce {
+        header "__pstl/cpu_algos/transform_reduce.h"
+      }
+    }
+    module dispatch           { header "__pstl/dispatch.h" }
+    module handle_exception   { header "__pstl/handle_exception.h" }
+  }
 
-module std_private_ostream_basic_ostream [system] {
-  header "__ostream/basic_ostream.h"
-  export std_streambuf
-}
-module std_private_ostream_print         [system] {
-  header "__ostream/print.h"
-  export std_print
-}
+  // Miscellaneous modules for top-level headers
+  module bit_reference_fwd {
+    header "__fwd/bit_reference.h"
+  }
+  module bit_reference {
+    header "__bit_reference"
+    export std.bit_reference_fwd
+  }
+  module hash_table           { header "__hash_table" }
+  module node_handle          { header "__node_handle" }
+  module split_buffer         { header "__split_buffer" }
+  module tree                 { header "__tree" }
+  module std_mbstate_t {
+    header "__std_mbstate_t.h"
+    export *
+  }
+  module verbose_abort {
+    header "__verbose_abort"
+  }
+  module internal_assert {
+    header "__assert"
+    export *
+  }
+
+  module undef_macros {
+    textual header "__undef_macros"
+  }
+
+  // This module needs to appear after __tree to work around issues with modules in Objective-C++ mode.
+  module coroutine {
+    module coroutine_handle         { header "__coroutine/coroutine_handle.h" }
+    module coroutine_traits         { header "__coroutine/coroutine_traits.h" }
+    module noop_coroutine_handle    { header "__coroutine/noop_coroutine_handle.h" }
+    module trivial_awaitables       { header "__coroutine/trivial_awaitables.h" }
 
-module std_private_random_bernoulli_distribution          [system] { header "__random/bernoulli_distribution.h" }
-module std_private_random_binomial_distribution           [system] { header "__random/binomial_distribution.h" }
-module std_private_random_cauchy_distribution             [system] { header "__random/cauchy_distribution.h" }
-module std_private_random_chi_squared_distribution        [system] { header "__random/chi_squared_distribution.h" }
-module std_private_random_clamp_to_integral               [system] { header "__random/clamp_to_integral.h" }
-module std_private_random_default_random_engine           [system] { header "__random/default_random_engine.h" }
-module std_private_random_discard_block_engine            [system] { header "__random/discard_block_engine.h" }
-module std_private_random_discrete_distribution           [system] {
-  header "__random/discrete_distribution.h"
+    header "coroutine"
+    export *
+  }
+} // module std
+
+// C compatibility headers
+//
+// These modules need to be their own top-level modules because they depend on the system-provided
+// headers (via include_next), which are then free to include other C headers provided by libc++.
+// If we group these headers in a single module, we would end up with circular dependencies.
+module std_complex_h [system] {
+  header "complex.h"
   export *
 }
-module std_private_random_exponential_distribution        [system] { header "__random/exponential_distribution.h" }
-module std_private_random_extreme_value_distribution      [system] { header "__random/extreme_value_distribution.h" }
-module std_private_random_fisher_f_distribution           [system] { header "__random/fisher_f_distribution.h" }
-module std_private_random_gamma_distribution              [system] { header "__random/gamma_distribution.h" }
-module std_private_random_generate_canonical              [system] { header "__random/generate_canonical.h" }
-module std_private_random_geometric_distribution          [system] { header "__random/geometric_distribution.h" }
-module std_private_random_independent_bits_engine         [system] { header "__random/independent_bits_engine.h" }
-module std_private_random_is_seed_sequence                [system] { header "__random/is_seed_sequence.h" }
-module std_private_random_is_valid                        [system] { header "__random/is_valid.h" }
-module std_private_random_knuth_b                         [system] { header "__random/knuth_b.h" }
-module std_private_random_linear_congruential_engine      [system] { header "__random/linear_congruential_engine.h" }
-module std_private_random_log2                            [system] { header "__random/log2.h" }
-module std_private_random_lognormal_distribution          [system] { header "__random/lognormal_distribution.h" }
-module std_private_random_mersenne_twister_engine         [system] { header "__random/mersenne_twister_engine.h" }
-module std_private_random_negative_binomial_distribution  [system] { header "__random/negative_binomial_distribution.h" }
-module std_private_random_normal_distribution             [system] { header "__random/normal_distribution.h" }
-module std_private_random_piecewise_constant_distribution [system] {
-  header "__random/piecewise_constant_distribution.h"
+module std_ctype_h [system] {
+  header "ctype.h"
   export *
 }
-module std_private_random_piecewise_linear_distribution   [system] {
-  header "__random/piecewise_linear_distribution.h"
+module std_errno_h [system] {
+  header "errno.h"
   export *
 }
-module std_private_random_poisson_distribution            [system] { header "__random/poisson_distribution.h" }
-module std_private_random_random_device                   [system] {
-  header "__random/random_device.h"
+module std_fenv_h [system] {
+  header "fenv.h"
   export *
 }
-module std_private_random_ranlux                          [system] { header "__random/ranlux.h" }
-module std_private_random_seed_seq                        [system] {
-  header "__random/seed_seq.h"
+module std_float_h [system] {
+  header "float.h"
   export *
 }
-module std_private_random_shuffle_order_engine            [system] { header "__random/shuffle_order_engine.h" }
-module std_private_random_student_t_distribution          [system] { header "__random/student_t_distribution.h" }
-module std_private_random_subtract_with_carry_engine      [system] { header "__random/subtract_with_carry_engine.h" }
-module std_private_random_uniform_int_distribution        [system] { header "__random/uniform_int_distribution.h" }
-module std_private_random_uniform_random_bit_generator    [system] { header "__random/uniform_random_bit_generator.h" }
-module std_private_random_uniform_real_distribution       [system] { header "__random/uniform_real_distribution.h" }
-module std_private_random_weibull_distribution            [system] { header "__random/weibull_distribution.h" }
-
-module std_private_ranges_access                     [system] { header "__ranges/access.h" }
-module std_private_ranges_all                        [system] {
-  header "__ranges/all.h"
-  export std_private_functional_compose
-  export std_private_functional_perfect_forward
-  export std_private_ranges_owning_view
-}
-module std_private_ranges_as_rvalue_view             [system] { header "__ranges/as_rvalue_view.h" }
-module std_private_ranges_chunk_by_view              [system] { header "__ranges/chunk_by_view.h" }
-module std_private_ranges_common_view                [system] { header "__ranges/common_view.h" }
-module std_private_ranges_concepts                   [system] {
-  header "__ranges/concepts.h"
-  export std_private_iterator_concepts
-}
-module std_private_ranges_container_compatible_range [system] { header "__ranges/container_compatible_range.h" }
-module std_private_ranges_counted                    [system] {
-  header "__ranges/counted.h"
-  export std_span
-}
-module std_private_ranges_dangling                   [system] { header "__ranges/dangling.h" }
-module std_private_ranges_data                       [system] { header "__ranges/data.h" }
-module std_private_ranges_drop_view                  [system] { header "__ranges/drop_view.h" }
-module std_private_ranges_drop_while_view            [system] { header "__ranges/drop_while_view.h" }
-module std_private_ranges_elements_view              [system] { header "__ranges/elements_view.h" }
-module std_private_ranges_empty                      [system] { header "__ranges/empty.h" }
-module std_private_ranges_empty_view                 [system] { header "__ranges/empty_view.h" }
-module std_private_ranges_enable_borrowed_range      [system] { header "__ranges/enable_borrowed_range.h" }
-module std_private_ranges_enable_view                [system] { header "__ranges/enable_view.h" }
-module std_private_ranges_filter_view                [system] {
-  header "__ranges/filter_view.h"
-  export std_private_ranges_range_adaptor
-}
-module std_private_ranges_from_range                 [system] { header "__ranges/from_range.h" }
-module std_private_ranges_iota_view                  [system] { header "__ranges/iota_view.h" }
-module std_private_ranges_istream_view               [system] {
-  header "__ranges/istream_view.h"
-}
-module std_private_ranges_join_view                  [system] {
-  header "__ranges/join_view.h"
-  export std_private_iterator_iterator_with_data
-  export std_private_iterator_segmented_iterator
-}
-module std_private_ranges_lazy_split_view            [system] {
-  header "__ranges/lazy_split_view.h"
-  export std_private_ranges_non_propagating_cache
+module std_inttypes_h [system] {
+  header "inttypes.h"
+  export *
 }
-module std_private_ranges_movable_box                [system] { header "__ranges/movable_box.h" }
-module std_private_ranges_non_propagating_cache      [system] { header "__ranges/non_propagating_cache.h" }
-module std_private_ranges_owning_view                [system] { header "__ranges/owning_view.h" }
-module std_private_ranges_range_adaptor              [system] { header "__ranges/range_adaptor.h" }
-module std_private_ranges_rbegin                     [system] { header "__ranges/rbegin.h" }
-module std_private_ranges_ref_view                   [system] { header "__ranges/ref_view.h" }
-module std_private_ranges_rend                       [system] { header "__ranges/rend.h" }
-module std_private_ranges_repeat_view                [system] { header "__ranges/repeat_view.h" }
-module std_private_ranges_reverse_view               [system] { header "__ranges/reverse_view.h" }
-module std_private_ranges_single_view                [system] { header "__ranges/single_view.h" }
-module std_private_ranges_size                       [system] {
-  header "__ranges/size.h"
-  export std_private_type_traits_make_unsigned
+module std_locale_h [system] {
+  header "locale.h"
+  export *
 }
-module std_private_ranges_split_view                 [system] { header "__ranges/split_view.h" }
-module std_private_ranges_subrange                   [system] {
-  header "__ranges/subrange.h"
-  export std_private_ranges_subrange_fwd
+module std_math_h [system] {
+  header "math.h"
+  export *
 }
-module std_private_ranges_subrange_fwd               [system] {
-  header "__fwd/subrange.h"
-  export std_private_iterator_concepts
+module std_stdatomic_h [system] {
+  header "stdatomic.h"
+  export *
 }
-module std_private_ranges_take_view                  [system] { header "__ranges/take_view.h" }
-module std_private_ranges_take_while_view            [system] { header "__ranges/take_while_view.h" }
-module std_private_ranges_to                         [system] { header "__ranges/to.h" }
-module std_private_ranges_transform_view             [system] {
-  header "__ranges/transform_view.h"
-  export std_private_functional_bind_back
-  export std_private_functional_perfect_forward
-  export std_private_ranges_movable_box
+module std_stdbool_h [system] {
+  // <stdbool.h>'s __bool_true_false_are_defined macro requires textual inclusion.
+  textual header "stdbool.h"
 }
-module std_private_ranges_view_interface             [system] { header "__ranges/view_interface.h" }
-module std_private_ranges_views                      [system] { header "__ranges/views.h" }
-module std_private_ranges_zip_view                   [system] {
-  header "__ranges/zip_view.h"
-  export std_private_utility_pair
+module std_stddef_h [system] {
+  // <stddef.h>'s __need_* macros require textual inclusion.
+  textual header "stddef.h"
 }
-
-module std_private_span_span_fwd [system] { header "__fwd/span.h" }
-
-module std_private_stack_fwd [system] { header "__fwd/stack.h" }
-
-module std_private_string_char_traits           [system] {
-  header "__string/char_traits.h"
+module std_stdint_h [system] {
+  header "stdint.h"
   export *
 }
-module std_private_string_constexpr_c_functions [system] {
-  header "__string/constexpr_c_functions.h"
-  export std_private_type_traits_is_equality_comparable
-}
-module std_private_string_extern_template_lists [system] { header "__string/extern_template_lists.h" }
-module std_private_string_string_fwd            [system] { header "__fwd/string.h" }
-
-module std_private_string_view_string_view_fwd [system] { header "__fwd/string_view.h" }
-
-module std_private_system_error_errc            [system] { header "__system_error/errc.h" }
-module std_private_system_error_error_category  [system] { header "__system_error/error_category.h" }
-module std_private_system_error_error_code      [system] {
-  header "__system_error/error_code.h"
-  export std_private_functional_hash
-  export std_private_functional_unary_function
+module std_stdio_h [system] {
+  // <stdio.h>'s __need_* macros require textual inclusion.
+  textual header "stdio.h"
 }
-module std_private_system_error_error_condition [system] {
-  header "__system_error/error_condition.h"
-  export std_private_functional_hash
-  export std_private_functional_unary_function
+module std_stdlib_h [system] {
+  // <stdlib.h>'s __need_* macros require textual inclusion.
+  textual header "stdlib.h"
 }
-module std_private_system_error_system_error    [system] { header "__system_error/system_error.h" }
-
-module std_private_thread_formatter            [system] { header "__thread/formatter.h" }
-module std_private_thread_id                   [system] { header "__thread/id.h" }
-module std_private_thread_jthread              [system] {
-  header "__thread/jthread.h"
+module std_string_h [system] {
+  header "string.h"
   export *
 }
-module std_private_thread_poll_with_backoff    [system] { header "__thread/poll_with_backoff.h" }
-module std_private_thread_support              [system] {
-  header "__thread/support.h"
+module std_tgmath_h [system] {
+  header "tgmath.h"
   export *
 }
-module std_private_thread_support_c11          [system] { textual header "__thread/support/c11.h" }
-module std_private_thread_support_external     [system] { textual header "__thread/support/external.h" }
-module std_private_thread_support_pthread      [system] { textual header "__thread/support/pthread.h" }
-module std_private_thread_support_windows      [system] { textual header "__thread/support/windows.h" }
-module std_private_thread_this_thread          [system] { header "__thread/this_thread.h" }
-module std_private_thread_thread               [system] {
-  header "__thread/thread.h"
+module std_uchar_h [system] {
+  header "uchar.h"
   export *
 }
-module std_private_thread_timed_backoff_policy [system] { header "__thread/timed_backoff_policy.h" }
-
-module std_private_tuple_find_index             [system] { header "__tuple/find_index.h" }
-module std_private_tuple_ignore                 [system] { header "__tuple/ignore.h" }
-module std_private_tuple_make_tuple_types       [system] { header "__tuple/make_tuple_types.h" }
-module std_private_tuple_tuple_like_no_subrange [system] {
-  header "__tuple/tuple_like_no_subrange.h"
-}
-module std_private_tuple_sfinae_helpers         [system] { header "__tuple/sfinae_helpers.h" }
-module std_private_tuple_tuple_element          [system] { header "__tuple/tuple_element.h" }
-module std_private_tuple_tuple_fwd              [system] { header "__fwd/tuple.h" }
-module std_private_get_fwd                      [system] {
-  header "__fwd/get.h"
-  export std_private_array_array_fwd
-  export std_private_complex_complex_fwd
-  export std_private_ranges_subrange_fwd
-  export std_private_tuple_tuple_fwd
-  export std_private_utility_pair_fwd
-  export std_private_variant_fwd
+module std_wchar_h [system] {
+  // <wchar.h>'s __need_* macros require textual inclusion.
+  textual header "wchar.h"
 }
-module std_private_tuple_tuple_indices          [system] { header "__tuple/tuple_indices.h" }
-module std_private_tuple_tuple_like             [system] {
-  header "__tuple/tuple_like.h"
+module std_wctype_h [system] {
+  header "wctype.h"
   export *
 }
-module std_private_tuple_tuple_like_ext         [system] { header "__tuple/tuple_like_ext.h" }
-module std_private_tuple_tuple_size             [system] {
-  header "__tuple/tuple_size.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_tuple_tuple_types            [system] { header "__tuple/tuple_types.h" }
 
-module std_private_type_traits_add_const                                 [system] { header "__type_traits/add_const.h" }
-module std_private_type_traits_add_cv                                    [system] { header "__type_traits/add_cv.h" }
-module std_private_type_traits_add_lvalue_reference                      [system] {
-  header "__type_traits/add_lvalue_reference.h"
-  export std_private_type_traits_is_referenceable
-}
-module std_private_type_traits_add_pointer                               [system] { header "__type_traits/add_pointer.h" }
-module std_private_type_traits_add_rvalue_reference                      [system] { header "__type_traits/add_rvalue_reference.h" }
-module std_private_type_traits_add_volatile                              [system] { header "__type_traits/add_volatile.h" }
-module std_private_type_traits_aligned_storage                           [system] { header "__type_traits/aligned_storage.h" }
-module std_private_type_traits_aligned_union                             [system] { header "__type_traits/aligned_union.h" }
-module std_private_type_traits_alignment_of                              [system] { header "__type_traits/alignment_of.h" }
-module std_private_type_traits_can_extract_key                           [system] { header "__type_traits/can_extract_key.h" }
-module std_private_type_traits_common_reference                          [system] {
-  header "__type_traits/common_reference.h"
-  export std_private_type_traits_remove_cvref
-}
-module std_private_type_traits_common_type                               [system] {
-  header "__type_traits/common_type.h"
-  export std_private_type_traits_type_identity
-  export std_private_utility_declval
-  export std_private_utility_empty
-}
-module std_private_type_traits_conditional                               [system] { header "__type_traits/conditional.h" }
-module std_private_type_traits_conjunction                               [system] { header "__type_traits/conjunction.h" }
-module std_private_type_traits_copy_cv                                   [system] { header "__type_traits/copy_cv.h" }
-module std_private_type_traits_copy_cvref                                [system] { header "__type_traits/copy_cvref.h" }
-module std_private_type_traits_datasizeof                                [system] { header "__type_traits/datasizeof.h" }
-module std_private_type_traits_decay                                     [system] {
-  header "__type_traits/decay.h"
-  export std_private_type_traits_add_pointer
-}
-module std_private_type_traits_dependent_type                            [system] { header "__type_traits/dependent_type.h" }
-module std_private_type_traits_desugars_to                               [system] { header "__type_traits/desugars_to.h" }
-module std_private_type_traits_disjunction                               [system] { header "__type_traits/disjunction.h" }
-module std_private_type_traits_enable_if                                 [system] { header "__type_traits/enable_if.h" }
-module std_private_type_traits_extent                                    [system] { header "__type_traits/extent.h" }
-module std_private_type_traits_has_unique_object_representation          [system] { header "__type_traits/has_unique_object_representation.h" }
-module std_private_type_traits_has_virtual_destructor                    [system] { header "__type_traits/has_virtual_destructor.h" }
-module std_private_type_traits_integral_constant                         [system] { header "__type_traits/integral_constant.h" }
-module std_private_type_traits_invoke                                    [system] {
-  header "__type_traits/invoke.h"
-  export std_private_type_traits_conditional
-  export std_private_type_traits_decay
-  export std_private_type_traits_decay
-  export std_private_type_traits_enable_if
-  export std_private_type_traits_is_base_of
-  export std_private_type_traits_is_core_convertible
-  export std_private_type_traits_is_reference_wrapper
-  export std_private_type_traits_is_same
-  export std_private_type_traits_is_void
-  export std_private_type_traits_nat
-  export std_private_type_traits_remove_cv
-}
-module std_private_type_traits_is_abstract                               [system] { header "__type_traits/is_abstract.h" }
-module std_private_type_traits_is_aggregate                              [system] { header "__type_traits/is_aggregate.h" }
-module std_private_type_traits_is_allocator                              [system] { header "__type_traits/is_allocator.h" }
-module std_private_type_traits_is_always_bitcastable                     [system] { header "__type_traits/is_always_bitcastable.h" }
-module std_private_type_traits_is_arithmetic                             [system] {
-  header "__type_traits/is_arithmetic.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_array                                  [system] {
-  header "__type_traits/is_array.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_assignable                             [system] { header "__type_traits/is_assignable.h" }
-module std_private_type_traits_is_base_of                                [system] { header "__type_traits/is_base_of.h" }
-module std_private_type_traits_is_bounded_array                          [system] { header "__type_traits/is_bounded_array.h" }
-module std_private_type_traits_is_callable                               [system] {
-  header "__type_traits/is_callable.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_char_like_type                         [system] { header "__type_traits/is_char_like_type.h" }
-module std_private_type_traits_is_class                                  [system] { header "__type_traits/is_class.h" }
-module std_private_type_traits_is_compound                               [system] { header "__type_traits/is_compound.h" }
-module std_private_type_traits_is_const                                  [system] { header "__type_traits/is_const.h" }
-module std_private_type_traits_is_constant_evaluated                     [system] { header "__type_traits/is_constant_evaluated.h" }
-module std_private_type_traits_is_constructible                          [system] { header "__type_traits/is_constructible.h" }
-module std_private_type_traits_is_convertible                            [system] {
-  header "__type_traits/is_convertible.h"
-  export std_private_type_traits_is_array
-}
-module std_private_type_traits_is_copy_assignable                        [system] { header "__type_traits/is_copy_assignable.h" }
-module std_private_type_traits_is_copy_constructible                     [system] { header "__type_traits/is_copy_constructible.h" }
-module std_private_type_traits_is_core_convertible                       [system] {
-  header "__type_traits/is_core_convertible.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_destructible                           [system] { header "__type_traits/is_destructible.h" }
-module std_private_type_traits_is_empty                                  [system] { header "__type_traits/is_empty.h" }
-module std_private_type_traits_is_enum                                   [system] {
-  header "__type_traits/is_enum.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_equality_comparable                    [system] {
-  header "__type_traits/is_equality_comparable.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_execution_policy                       [system] {
-  header "__type_traits/is_execution_policy.h"
-  export std_private_type_traits_remove_cvref
-}
-module std_private_type_traits_is_final                                  [system] { header "__type_traits/is_final.h" }
-module std_private_type_traits_is_floating_point                         [system] { header "__type_traits/is_floating_point.h" }
-module std_private_type_traits_is_function                               [system] { header "__type_traits/is_function.h" }
-module std_private_type_traits_is_fundamental                            [system] { header "__type_traits/is_fundamental.h" }
-module std_private_type_traits_is_implicitly_default_constructible       [system] {
-  header "__type_traits/is_implicitly_default_constructible.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_integral                               [system] {
-  header "__type_traits/is_integral.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_literal_type                           [system] { header "__type_traits/is_literal_type.h" }
-module std_private_type_traits_is_member_pointer                         [system] { header "__type_traits/is_member_pointer.h" }
-module std_private_type_traits_is_nothrow_assignable                     [system] { header "__type_traits/is_nothrow_assignable.h" }
-module std_private_type_traits_is_nothrow_constructible                  [system] {
-  header "__type_traits/is_nothrow_constructible.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_nothrow_convertible                    [system] { header "__type_traits/is_nothrow_convertible.h" }
-module std_private_type_traits_is_nothrow_destructible                   [system] {
-  header "__type_traits/is_nothrow_destructible.h"
-  export std_private_type_traits_is_destructible
-}
-module std_private_type_traits_is_null_pointer                           [system] {
-  header "__type_traits/is_null_pointer.h"
-  export std_cstddef
-}
-module std_private_type_traits_is_object                                 [system] {
-  header "__type_traits/is_object.h"
-  export std_private_type_traits_is_scalar
-}
-module std_private_type_traits_is_pod                                    [system] { header "__type_traits/is_pod.h" }
-module std_private_type_traits_is_pointer                                [system] { header "__type_traits/is_pointer.h" }
-module std_private_type_traits_is_polymorphic                            [system] { header "__type_traits/is_polymorphic.h" }
-module std_private_type_traits_is_primary_template                       [system] {
-  header "__type_traits/is_primary_template.h"
-  export std_private_type_traits_enable_if
-}
-module std_private_type_traits_is_reference                              [system] {
-  header "__type_traits/is_reference.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_reference_wrapper                      [system] { header "__type_traits/is_reference_wrapper.h" }
-module std_private_type_traits_is_referenceable                          [system] { header "__type_traits/is_referenceable.h" }
-module std_private_type_traits_is_same                                   [system] {
-  header "__type_traits/is_same.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_scalar                                 [system] {
-  header "__type_traits/is_scalar.h"
-  export std_private_type_traits_is_null_pointer
-}
-module std_private_type_traits_is_signed                                 [system] { header "__type_traits/is_signed.h" }
-module std_private_type_traits_is_signed_integer                         [system] { header "__type_traits/is_signed_integer.h" }
-module std_private_type_traits_is_specialization                         [system] { header "__type_traits/is_specialization.h" }
-module std_private_type_traits_is_standard_layout                        [system] { header "__type_traits/is_standard_layout.h" }
-module std_private_type_traits_is_swappable                              [system] {
-  header "__type_traits/is_swappable.h"
-  export std_private_type_traits_is_move_constructible
-}
-module std_private_type_traits_is_trivial                                [system] { header "__type_traits/is_trivial.h" }
-module std_private_type_traits_is_trivially_assignable                   [system] { header "__type_traits/is_trivially_assignable.h" }
-module std_private_type_traits_is_trivially_constructible                [system] { header "__type_traits/is_trivially_constructible.h" }
-module std_private_type_traits_is_trivially_copyable                     [system] {
-  header "__type_traits/is_trivially_copyable.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_trivially_destructible                 [system] { header "__type_traits/is_trivially_destructible.h" }
-module std_private_type_traits_is_trivially_lexicographically_comparable [system] { header "__type_traits/is_trivially_lexicographically_comparable.h" }
-module std_private_type_traits_is_trivially_relocatable                  [system] { header "__type_traits/is_trivially_relocatable.h" }
-module std_private_type_traits_is_unbounded_array                        [system] { header "__type_traits/is_unbounded_array.h" }
-module std_private_type_traits_is_union                                  [system] { header "__type_traits/is_union.h" }
-module std_private_type_traits_is_unsigned                               [system] { header "__type_traits/is_unsigned.h" }
-module std_private_type_traits_is_unsigned_integer                       [system] { header "__type_traits/is_unsigned_integer.h" }
-module std_private_type_traits_is_valid_expansion                        [system] { header "__type_traits/is_valid_expansion.h" }
-module std_private_type_traits_is_void                                   [system] {
-  header "__type_traits/is_void.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_type_traits_is_volatile                               [system] { header "__type_traits/is_volatile.h" }
-module std_private_type_traits_lazy                                      [system] { header "__type_traits/lazy.h" }
-module std_private_type_traits_make_32_64_or_128_bit                     [system] { header "__type_traits/make_32_64_or_128_bit.h" }
-module std_private_type_traits_make_const_lvalue_ref                     [system] { header "__type_traits/make_const_lvalue_ref.h" }
-module std_private_type_traits_make_signed                               [system] { header "__type_traits/make_signed.h" }
-module std_private_type_traits_make_unsigned                             [system] {
-  header "__type_traits/make_unsigned.h"
-  export std_private_type_traits_is_unsigned
-}
-module std_private_type_traits_maybe_const                               [system] { header "__type_traits/maybe_const.h" }
-module std_private_type_traits_nat                                       [system] { header "__type_traits/nat.h" }
-module std_private_type_traits_negation                                  [system] { header "__type_traits/negation.h" }
-module std_private_type_traits_promote                                   [system] { header "__type_traits/promote.h" }
-module std_private_type_traits_rank                                      [system] { header "__type_traits/rank.h" }
-module std_private_type_traits_remove_all_extents                        [system] { header "__type_traits/remove_all_extents.h" }
-module std_private_type_traits_remove_const                              [system] { header "__type_traits/remove_const.h" }
-module std_private_type_traits_remove_const_ref                          [system] { header "__type_traits/remove_const_ref.h" }
-module std_private_type_traits_remove_cv                                 [system] {
-  header "__type_traits/remove_cv.h"
-  export std_private_type_traits_remove_const
-  export std_private_type_traits_remove_volatile
-}
-module std_private_type_traits_remove_cvref                              [system] { header "__type_traits/remove_cvref.h" }
-module std_private_type_traits_remove_extent                             [system] { header "__type_traits/remove_extent.h" }
-module std_private_type_traits_remove_pointer                            [system] { header "__type_traits/remove_pointer.h" }
-module std_private_type_traits_remove_reference                          [system] { header "__type_traits/remove_reference.h" }
-module std_private_type_traits_remove_volatile                           [system] { header "__type_traits/remove_volatile.h" }
-module std_private_type_traits_result_of                                 [system] { header "__type_traits/result_of.h" }
-module std_private_type_traits_strip_signature                           [system] { header "__type_traits/strip_signature.h" }
-module std_private_type_traits_type_identity                             [system] { header "__type_traits/type_identity.h" }
-module std_private_type_traits_type_list                                 [system] { header "__type_traits/type_list.h" }
-module std_private_type_traits_underlying_type                           [system] {
-  header "__type_traits/underlying_type.h"
-  export std_private_type_traits_is_enum
-}
-module std_private_type_traits_unwrap_ref                                [system] { header "__type_traits/unwrap_ref.h" }
-module std_private_type_traits_void_t                                    [system] { header "__type_traits/void_t.h" }
-
-module std_private_utility_as_const               [system] { header "__utility/as_const.h" }
-module std_private_utility_as_lvalue              [system] { header "__utility/as_lvalue.h" }
-module std_private_utility_auto_cast              [system] {
-  header "__utility/auto_cast.h"
-  export std_private_type_traits_decay
-}
-module std_private_utility_cmp                    [system] {
-  header "__utility/cmp.h"
-  export std_private_type_traits_make_unsigned
-}
-module std_private_utility_convert_to_integral    [system] { header "__utility/convert_to_integral.h" }
-module std_private_utility_declval                [system] { header "__utility/declval.h" }
-module std_private_utility_empty                  [system] { header "__utility/empty.h" }
-module std_private_utility_exception_guard        [system] { header "__utility/exception_guard.h" }
-module std_private_utility_exchange               [system] { header "__utility/exchange.h" }
-module std_private_utility_forward                [system] { header "__utility/forward.h" }
-module std_private_utility_forward_like           [system] { header "__utility/forward_like.h" }
-module std_private_utility_in_place               [system] {
-  header "__utility/in_place.h"
-  export std_private_type_traits_integral_constant
-}
-module std_private_utility_integer_sequence       [system] { header "__utility/integer_sequence.h" }
-module std_private_utility_is_pointer_in_range    [system] { header "__utility/is_pointer_in_range.h" }
-module std_private_utility_is_valid_range         [system] { header "__utility/is_valid_range.h" }
-module std_private_utility_move                   [system] {
-  header "__utility/move.h"
-  export std_private_type_traits_is_copy_constructible
-  export std_private_type_traits_is_nothrow_move_constructible
-  export std_private_type_traits_remove_reference
-}
-module std_private_utility_no_destroy             [system] { header "__utility/no_destroy.h" }
-module std_private_utility_pair                   [system] {
-  header "__utility/pair.h"
-  export std_private_ranges_subrange_fwd
-  export std_private_tuple_pair_like
-  export std_private_type_traits_is_assignable
-  export std_private_type_traits_is_constructible
-  export std_private_type_traits_is_convertible
-  export std_private_type_traits_is_copy_assignable
-  export std_private_type_traits_is_move_assignable
-  export std_private_type_traits_is_nothrow_copy_constructible
-  export std_private_type_traits_is_nothrow_default_constructible
-  export std_private_type_traits_is_nothrow_move_assignable
-  export std_private_utility_pair_fwd
-}
-module std_private_utility_pair_fwd                [system] { header "__fwd/pair.h" }
-module std_private_utility_piecewise_construct     [system] { header "__utility/piecewise_construct.h" }
-module std_private_utility_priority_tag            [system] { header "__utility/priority_tag.h" }
-module std_private_utility_private_constructor_tag [system] { header "__utility/private_constructor_tag.h" }
-module std_private_utility_rel_ops                 [system] { header "__utility/rel_ops.h" }
-module std_private_utility_small_buffer            [system] { header "__utility/small_buffer.h" }
-module std_private_utility_swap                    [system] {
-  header "__utility/swap.h"
-  export std_private_type_traits_is_swappable
+// This header is used by other C compatibility headers so it needs to be in its own module.
+module std_private_mbstate_t [system] {
+  header "__mbstate_t.h"
+  export *
 }
-module std_private_utility_to_underlying           [system] { header "__utility/to_underlying.h" }
-module std_private_utility_unreachable             [system] { header "__utility/unreachable.h" }
-
-module std_private_variant_monostate               [system] { header "__variant/monostate.h" }
-module std_private_variant_fwd                     [system] { header "__fwd/variant.h" }
-
-module std_private_vector_fwd                      [system] { header "__fwd/vector.h" }
diff --git a/libcxx/test/libcxx/clang_modules_include.gen.py b/libcxx/test/libcxx/clang_modules_include.gen.py
index f0421b2e73813..bc028f2a0809a 100644
--- a/libcxx/test/libcxx/clang_modules_include.gen.py
+++ b/libcxx/test/libcxx/clang_modules_include.gen.py
@@ -37,13 +37,17 @@
 // TODO: Investigate this failure
 // UNSUPPORTED: LIBCXX-FREEBSD-FIXME
 
+// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored
+// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
+
 {lit_header_restrictions.get(header, '')}
 
 #include <{header}>
 """)
 
-print(f"""\
-//--- __std_clang_module.compile.pass.mm
+print(
+    f"""\
+//--- import_std.compile.pass.mm
 // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only
 
 // REQUIRES: clang-modules-build
@@ -61,6 +65,10 @@
 // TODO: Investigate this failure
 // UNSUPPORTED: LIBCXX-FREEBSD-FIXME
 
+// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored
+// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
+
 @import std;
 
-""")
+"""
+)
diff --git a/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
index 9147ca93866b2..cea074a4e70f1 100644
--- a/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
@@ -8,6 +8,10 @@
 
 // UNSUPPORTED: libcpp-has-abi-fix-unordered-container-size-type, libcpp-abi-no-compressed-pair-padding
 
+// std::unique_ptr is used as an implementation detail of the unordered containers, so the layout of
+// unordered containers changes when bounded unique_ptr is enabled.
+// UNSUPPORTED: libcpp-has-abi-bounded-unique_ptr
+
 #include <cstdint>
 #include <unordered_map>
 
diff --git a/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
index dc6cc082c3b99..2a112aff227d8 100644
--- a/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
@@ -8,6 +8,10 @@
 
 // UNSUPPORTED: libcpp-has-abi-fix-unordered-container-size-type, libcpp-abi-no-compressed-pair-padding
 
+// std::unique_ptr is used as an implementation detail of the unordered containers, so the layout of
+// unordered containers changes when bounded unique_ptr is enabled.
+// UNSUPPORTED: libcpp-has-abi-bounded-unique_ptr
+
 #include <cstdint>
 #include <unordered_set>
 
diff --git a/libcxx/test/std/experimental/utilities/utility/utility.synop/includes.pass.cpp b/libcxx/test/std/experimental/utilities/utility/utility.synop/includes.pass.cpp
deleted file mode 100644
index 7e27adfab1971..0000000000000
--- a/libcxx/test/std/experimental/utilities/utility/utility.synop/includes.pass.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <experimental/utility>
-
-#include <experimental/utility>
-
-#include "test_macros.h"
-
-#ifndef _LIBCPP_UTILITY
-#   error "<experimental/utility> must include <utility>"
-#endif
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/incomplete.sh.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/incomplete.sh.cpp
new file mode 100644
index 0000000000000..4a03d2bcf07bf
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/incomplete.sh.cpp
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <memory>
+
+// unique_ptr
+
+// Make sure that we can form unique_ptrs to incomplete types and perform restricted
+// operations on them. This requires setting up a TU where the type is complete and
+// the unique_ptr is created and destroyed, and a TU where the type is incomplete and
+// we check that a restricted set of operations can be performed on the unique_ptr.
+
+// RUN: %{cxx} %s %{flags} %{compile_flags} -c -o %t.tu1.o -DCOMPLETE
+// RUN: %{cxx} %s %{flags} %{compile_flags} -c -o %t.tu2.o -DINCOMPLETE
+// RUN: %{cxx} %t.tu1.o %t.tu2.o %{flags} %{link_flags} -o %t.exe
+// RUN: %{exec} %t.exe
+
+#include <memory>
+#include <cassert>
+
+struct T;
+extern void use(std::unique_ptr<T>& ptr);
+extern void use(std::unique_ptr<T[]>& ptr);
+
+#ifdef INCOMPLETE
+
+void use(std::unique_ptr<T>& ptr) {
+  {
+    T* x = ptr.get();
+    assert(x != nullptr);
+  }
+  {
+    T& ref = *ptr;
+    assert(&ref == ptr.get());
+  }
+  {
+    bool engaged = static_cast<bool>(ptr);
+    assert(engaged);
+  }
+  {
+    assert(ptr == ptr);
+    assert(!(ptr != ptr));
+    assert(!(ptr < ptr));
+    assert(!(ptr > ptr));
+    assert(ptr <= ptr);
+    assert(ptr >= ptr);
+  }
+}
+
+void use(std::unique_ptr<T[]>& ptr) {
+  {
+    T* x = ptr.get();
+    assert(x != nullptr);
+  }
+  {
+    bool engaged = static_cast<bool>(ptr);
+    assert(engaged);
+  }
+  {
+    assert(ptr == ptr);
+    assert(!(ptr != ptr));
+    assert(!(ptr < ptr));
+    assert(!(ptr > ptr));
+    assert(ptr <= ptr);
+    assert(ptr >= ptr);
+  }
+}
+
+#endif // INCOMPLETE
+
+#ifdef COMPLETE
+
+struct T {}; // complete the type
+
+int main(int, char**) {
+  {
+    std::unique_ptr<T> ptr(new T());
+    use(ptr);
+  }
+
+  {
+    std::unique_ptr<T[]> ptr(new T[3]());
+    use(ptr);
+  }
+  return 0;
+}
+
+#endif // COMPLETE
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
new file mode 100644
index 0000000000000..1eaf2d5900356
--- /dev/null
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp
@@ -0,0 +1,166 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: has-unix-headers
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+// UNSUPPORTED: libcpp-hardening-mode=none
+// XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
+
+// <memory>
+//
+// unique_ptr<T[]>
+//
+// T& operator[](std::size_t);
+
+// This test ensures that we catch an out-of-bounds access in std::unique_ptr<T[]>::operator[]
+// when unique_ptr has the appropriate ABI configuration.
+
+#include <memory>
+#include <cstddef>
+#include <string>
+
+#include "check_assertion.h"
+#include "type_algorithms.h"
+
+struct MyDeleter {
+  MyDeleter() = default;
+
+  // required to exercise converting move-constructor
+  template <class T>
+  MyDeleter(std::default_delete<T> const&) {}
+
+  // required to exercise converting move-assignment
+  template <class T>
+  MyDeleter& operator=(std::default_delete<T> const&) {
+    return *this;
+  }
+
+  template <class T>
+  void operator()(T* ptr) const {
+    delete[] ptr;
+  }
+};
+
+template <class WithCookie, class NoCookie>
+void test() {
+  // For types with an array cookie, we can always detect OOB accesses.
+  {
+    // Check with the default deleter
+    {
+      {
+        std::unique_ptr<WithCookie[]> ptr(new WithCookie[5]);
+        TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
+      }
+      {
+        std::unique_ptr<WithCookie[]> ptr = std::make_unique<WithCookie[]>(5);
+        TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
+      }
+#if TEST_STD_VER >= 20
+      {
+        std::unique_ptr<WithCookie[]> ptr = std::make_unique_for_overwrite<WithCookie[]>(5);
+        TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = WithCookie(), "unique_ptr<T[]>::operator[](index): index out of range");
+      }
+#endif
+    }
+
+    // Check with a custom deleter
+    {
+      std::unique_ptr<WithCookie[], MyDeleter> ptr(new WithCookie[5]);
+      TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+  }
+
+  // For types that don't have an array cookie, things are a bit more complicated. We can detect OOB accesses
+  // only when the unique_ptr is created via an API where the size is passed down to the library so that we
+  // can store it inside the unique_ptr. That requires the appropriate ABI configuration to be enabled.
+  //
+  // Note that APIs that allow the size to be passed down to the library only support the default deleter
+  // as of writing this test.
+#if defined(_LIBCPP_ABI_BOUNDED_UNIQUE_PTR)
+  {
+    {
+      std::unique_ptr<NoCookie[]> ptr = std::make_unique<NoCookie[]>(5);
+      TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+#  if TEST_STD_VER >= 20
+    {
+      std::unique_ptr<NoCookie[]> ptr = std::make_unique_for_overwrite<NoCookie[]>(5);
+      TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = NoCookie(), "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+#  endif
+  }
+#endif
+
+  // Make sure that we carry the bounds information properly through conversions, assignments, etc.
+  // These tests are mostly relevant when the ABI setting is enabled (with a stateful bounds-checker),
+  // but we still run them for types with an array cookie either way.
+#if defined(_LIBCPP_ABI_BOUNDED_UNIQUE_PTR)
+  using Types = types::type_list<NoCookie, WithCookie>;
+#else
+  using Types = types::type_list<WithCookie>;
+#endif
+  types::for_each(Types(), []<class T> {
+    // Bounds carried through move construction
+    {
+      std::unique_ptr<T[]> ptr = std::make_unique<T[]>(5);
+      std::unique_ptr<T[]> other(std::move(ptr));
+      TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+
+    // Bounds carried through move assignment
+    {
+      std::unique_ptr<T[]> ptr = std::make_unique<T[]>(5);
+      std::unique_ptr<T[]> other;
+      other = std::move(ptr);
+      TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+
+    // Bounds carried through converting move-constructor
+    {
+      std::unique_ptr<T[]> ptr = std::make_unique<T[]>(5);
+      std::unique_ptr<T[], MyDeleter> other(std::move(ptr));
+      TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+
+    // Bounds carried through converting move-assignment
+    {
+      std::unique_ptr<T[]> ptr = std::make_unique<T[]>(5);
+      std::unique_ptr<T[], MyDeleter> other;
+      other = std::move(ptr);
+      TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr<T[]>::operator[](index): index out of range");
+    }
+  });
+}
+
+template <std::size_t Size>
+struct NoCookie {
+  char padding[Size];
+};
+
+template <std::size_t Size>
+struct WithCookie {
+  WithCookie() = default;
+  WithCookie(WithCookie const&) {}
+  WithCookie& operator=(WithCookie const&) { return *this; }
+  ~WithCookie() {}
+  char padding[Size];
+};
+
+int main(int, char**) {
+  test<WithCookie<1>, NoCookie<1>>();
+  test<WithCookie<2>, NoCookie<2>>();
+  test<WithCookie<3>, NoCookie<3>>();
+  test<WithCookie<4>, NoCookie<4>>();
+  test<WithCookie<8>, NoCookie<8>>();
+  test<WithCookie<16>, NoCookie<16>>();
+  test<WithCookie<32>, NoCookie<32>>();
+  test<WithCookie<256>, NoCookie<256>>();
+  test<std::string, int>();
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/get.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/get.pass.cpp
index 3bd3788960e2a..c92c39c8f299e 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/get.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/get.pass.cpp
@@ -10,43 +10,114 @@
 
 // unique_ptr
 
-// test get
+// pointer unique_ptr<T>::get() const noexcept;
+// pointer unique_ptr<T[]>::get() const noexcept;
 
 #include <memory>
 #include <cassert>
+#include <cstddef>
 
 #include "test_macros.h"
-#include "unique_ptr_test_helper.h"
 
-template <bool IsArray>
+template <class T>
 TEST_CONSTEXPR_CXX23 void test_basic() {
-  typedef typename std::conditional<IsArray, int[], int>::type VT;
-  typedef const VT CVT;
+  // non-const element type
   {
-    typedef std::unique_ptr<VT> U;
-    int* p = newValue<VT>(1);
-    U s(p);
-    U const& sc = s;
-    ASSERT_SAME_TYPE(decltype(s.get()), int*);
-    ASSERT_SAME_TYPE(decltype(sc.get()), int*);
-    assert(s.get() == p);
-    assert(sc.get() == s.get());
+    // non-const access
+    {
+      T* x = new T;
+      std::unique_ptr<T> ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T*);
+      ASSERT_NOEXCEPT(ptr.get());
+      assert(ptr.get() == x);
+    }
+
+    // const access
+    {
+      T* x = new T;
+      std::unique_ptr<T> const ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T*);
+      ASSERT_NOEXCEPT(ptr.get());
+      assert(ptr.get() == x);
+    }
+  }
+
+  // const element type
+  {
+    // non-const access
+    {
+      T* x = new T;
+      std::unique_ptr<T const> ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T const*);
+      assert(ptr.get() == x);
+    }
+
+    // const access
+    {
+      T* x = new T;
+      std::unique_ptr<T const> const ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T const*);
+      assert(ptr.get() == x);
+    }
+  }
+
+  // Same thing but for unique_ptr<T[]>
+  // non-const element type
+  {
+    // non-const access
+    {
+      T* x = new T[3];
+      std::unique_ptr<T[]> ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T*);
+      ASSERT_NOEXCEPT(ptr.get());
+      assert(ptr.get() == x);
+    }
+
+    // const access
+    {
+      T* x = new T[3];
+      std::unique_ptr<T[]> const ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T*);
+      ASSERT_NOEXCEPT(ptr.get());
+      assert(ptr.get() == x);
+    }
   }
+
+  // const element type
   {
-    typedef std::unique_ptr<CVT> U;
-    const int* p = newValue<VT>(1);
-    U s(p);
-    U const& sc = s;
-    ASSERT_SAME_TYPE(decltype(s.get()), const int*);
-    ASSERT_SAME_TYPE(decltype(sc.get()), const int*);
-    assert(s.get() == p);
-    assert(sc.get() == s.get());
+    // non-const access
+    {
+      T* x = new T[3];
+      std::unique_ptr<T const[]> ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T const*);
+      assert(ptr.get() == x);
+    }
+
+    // const access
+    {
+      T* x = new T[3];
+      std::unique_ptr<T const[]> const ptr(x);
+      ASSERT_SAME_TYPE(decltype(ptr.get()), T const*);
+      assert(ptr.get() == x);
+    }
   }
 }
 
+template <std::size_t Size>
+struct WithSize {
+  char padding[Size];
+};
+
 TEST_CONSTEXPR_CXX23 bool test() {
-  test_basic</*IsArray*/ false>();
-  test_basic<true>();
+  test_basic<char>();
+  test_basic<int>();
+  test_basic<WithSize<1> >();
+  test_basic<WithSize<2> >();
+  test_basic<WithSize<3> >();
+  test_basic<WithSize<4> >();
+  test_basic<WithSize<8> >();
+  test_basic<WithSize<16> >();
+  test_basic<WithSize<256> >();
 
   return true;
 }
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
index fbb4dbc6e0308..ebfad8ec724e5 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/op_subscript.runtime.pass.cpp
@@ -10,51 +10,117 @@
 
 // unique_ptr
 
-// test op[](size_t)
+// T& unique_ptr::operator[](size_t) const
 
 #include <memory>
 #include <cassert>
-
-// TODO: Move TEST_IS_CONSTANT_EVALUATED into its own header
 #include <type_traits>
+#include <array>
 
 #include "test_macros.h"
+#include "type_algorithms.h"
 
-class A {
-  int state_;
-  static int next_;
+static int next = 0;
+struct EnumeratedDefaultCtor {
+  EnumeratedDefaultCtor() : value(0) { value = ++next; }
+  int value;
+};
 
-public:
-  TEST_CONSTEXPR_CXX23 A() : state_(0) {
-    if (!TEST_IS_CONSTANT_EVALUATED)
-      state_ = ++next_;
+template <std::size_t Size>
+struct WithTrivialDtor {
+  std::array<char, Size> padding = {'x'};
+  TEST_CONSTEXPR_CXX23 friend bool operator==(WithTrivialDtor const& x, WithTrivialDtor const& y) {
+    return x.padding == y.padding;
   }
+};
 
-  TEST_CONSTEXPR_CXX23 int get() const { return state_; }
-
-  friend TEST_CONSTEXPR_CXX23 bool operator==(const A& x, int y) { return x.state_ == y; }
-
-  TEST_CONSTEXPR_CXX23 A& operator=(int i) {
-    state_ = i;
-    return *this;
+template <std::size_t Size>
+struct WithNonTrivialDtor {
+  std::array<char, Size> padding = {'x'};
+  TEST_CONSTEXPR_CXX23 friend bool operator==(WithNonTrivialDtor const& x, WithNonTrivialDtor const& y) {
+    return x.padding == y.padding;
   }
+  TEST_CONSTEXPR_CXX23 ~WithNonTrivialDtor() {}
 };
 
-int A::next_ = 0;
+template <class T>
+struct CustomDeleter : std::default_delete<T> {};
 
 TEST_CONSTEXPR_CXX23 bool test() {
-  std::unique_ptr<A[]> p(new A[3]);
-  if (!TEST_IS_CONSTANT_EVALUATED) {
-    assert(p[0] == 1);
-    assert(p[1] == 2);
-    assert(p[2] == 3);
+  // Basic test
+  {
+    std::unique_ptr<int[]> p(new int[3]);
+    {
+      int& result = p[0];
+      result      = 0;
+    }
+    {
+      int& result = p[1];
+      result      = 1;
+    }
+    {
+      int& result = p[2];
+      result      = 2;
+    }
+
+    assert(p[0] == 0);
+    assert(p[1] == 1);
+    assert(p[2] == 2);
+  }
+
+  // Ensure that the order of access is correct after initializing a unique_ptr but
+  // before actually modifying any of its elements. The implementation would have to
+  // really try for this not to be the case, but we still check it.
+  //
+  // This requires assigning known values to the elements when they are first constructed,
+  // which requires global state.
+  {
+    if (!TEST_IS_CONSTANT_EVALUATED) {
+      std::unique_ptr<EnumeratedDefaultCtor[]> p(new EnumeratedDefaultCtor[3]);
+      assert(p[0].value == 1);
+      assert(p[1].value == 2);
+      assert(p[2].value == 3);
+    }
+  }
+
+  // Make sure operator[] is const-qualified
+  {
+    std::unique_ptr<int[]> const p(new int[3]);
+    p[0] = 42;
+    assert(p[0] == 42);
+  }
+
+  // Make sure we properly handle types with trivial and non-trivial destructors of different
+  // sizes. This is relevant because some implementations may want to use properties of the
+  // ABI like array cookies and these properties often depend on e.g. the triviality of T's
+  // destructor, T's size and so on.
+#if TEST_STD_VER >= 20 // this test is too painful to write before C++20
+  {
+    using TrickyCookieTypes = types::type_list<
+        WithTrivialDtor<1>,
+        WithTrivialDtor<2>,
+        WithTrivialDtor<3>,
+        WithTrivialDtor<4>,
+        WithTrivialDtor<8>,
+        WithTrivialDtor<16>,
+        WithTrivialDtor<256>,
+        WithNonTrivialDtor<1>,
+        WithNonTrivialDtor<2>,
+        WithNonTrivialDtor<3>,
+        WithNonTrivialDtor<4>,
+        WithNonTrivialDtor<8>,
+        WithNonTrivialDtor<16>,
+        WithNonTrivialDtor<256>>;
+    types::for_each(TrickyCookieTypes(), []<class T> {
+      types::for_each(types::type_list<std::default_delete<T[]>, CustomDeleter<T[]>>(), []<class Deleter> {
+        std::unique_ptr<T[], Deleter> p(new T[3]);
+        assert(p[0] == T());
+        assert(p[1] == T());
+        assert(p[2] == T());
+      });
+    });
   }
-  p[0] = 3;
-  p[1] = 2;
-  p[2] = 1;
-  assert(p[0] == 3);
-  assert(p[1] == 2);
-  assert(p[2] == 1);
+#endif // C++20
 
   return true;
 }
diff --git a/libcxx/utils/CMakeLists.txt b/libcxx/utils/CMakeLists.txt
index 1116531fa0653..027e485fc15ef 100644
--- a/libcxx/utils/CMakeLists.txt
+++ b/libcxx/utils/CMakeLists.txt
@@ -2,10 +2,6 @@ add_custom_target(libcxx-generate-feature-test-macros
     COMMAND "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/generate_feature_test_macro_components.py"
     COMMENT "Generate the <version> header and tests for feature test macros.")
 
-add_custom_target(libcxx-generate-std-clang-module-header
-  COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_clang_module_header.py"
-  COMMENT "Generate the <__std_clang_module> header")
-
 add_custom_target(libcxx-generate-std-cppm-in-file
   COMMAND
         "${Python3_EXECUTABLE}"
@@ -57,7 +53,6 @@ add_custom_target(libcxx-indic-conjunct-break-table
 
 add_custom_target(libcxx-generate-files
     DEPENDS libcxx-generate-feature-test-macros
-            libcxx-generate-std-clang-module-header
             libcxx-generate-std-cppm-in-file
             libcxx-generate-std-compat-cppm-in-file
             libcxx-generate-extended-grapheme-cluster-tables
diff --git a/libcxx/utils/generate_std_clang_module_header.py b/libcxx/utils/generate_std_clang_module_header.py
deleted file mode 100644
index 33c9acf395379..0000000000000
--- a/libcxx/utils/generate_std_clang_module_header.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# ===----------------------------------------------------------------------===##
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-# ===----------------------------------------------------------------------===##
-
-import os.path
-
-import libcxx.header_information
-
-header_restrictions = libcxx.header_information.header_restrictions
-
-libcxx_include_directory = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "include"
-)
-with open(
-    os.path.join(libcxx_include_directory, "__std_clang_module"), "w"
-) as std_clang_module_header:
-    std_clang_module_header.write(
-        """\
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// WARNING, this entire header is generated by
-// utils/generate_std_clang_module_header.py
-// DO NOT MODIFY!
-
-// This header should not be directly included, it's exclusively to import all
-// of the libc++ public clang modules for the `std` clang module to export. In
-// other words, it's to facilitate `@import std;` in Objective-C++ and `import std`
-// in Swift to expose all of the libc++ interfaces. This is generally not
-// recommended, however there are some clients that need to import all of libc++
-// without knowing what "all" is.
-#if !__building_module(std)
-#  error "Do not include this header directly, include individual headers instead"
-#endif
-
-#include <__config>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-"""
-    )
-    # Include the angle brackets in sorting so that <a.h> sorts before <a>
-    # like check-format wants.
-    for include, header in sorted([(f"<{header}>", header) for header in libcxx.header_information.public_headers]):
-        header_restriction = header_restrictions.get(header)
-        if header_restriction:
-            std_clang_module_header.write(f"#if {header_restriction}\n")
-            std_clang_module_header.write(f"#  include {include}\n")
-            std_clang_module_header.write(f"#endif\n")
-        else:
-            std_clang_module_header.write(f"#include {include}\n")
diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py
index 15456171b5483..29810c8ffee53 100644
--- a/libcxx/utils/libcxx/test/features.py
+++ b/libcxx/utils/libcxx/test/features.py
@@ -374,6 +374,7 @@ def _mingwSupportsModules(cfg):
     "_LIBCPP_ABI_BOUNDED_ITERATORS": "libcpp-has-abi-bounded-iterators",
     "_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STRING": "libcpp-has-abi-bounded-iterators-in-string",
     "_LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR": "libcpp-has-abi-bounded-iterators-in-vector",
+    "_LIBCPP_ABI_BOUNDED_UNIQUE_PTR": "libcpp-has-abi-bounded-unique_ptr",
     "_LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE": "libcpp-has-abi-fix-unordered-container-size-type",
     "_LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR": "libcpp-deprecated-abi-disable-pair-trivial-copy-ctor",
     "_LIBCPP_ABI_NO_COMPRESSED_PAIR_PADDING": "libcpp-abi-no-compressed-pair-padding",
@@ -609,10 +610,17 @@ def check_gdb(cfg):
 # Helpers to define correspondances between LLVM versions and vendor system versions.
 # Those are used for backdeployment features below, do not use directly in tests.
 DEFAULT_FEATURES += [
+    Feature(
+        name="_target-has-llvm-18",
+        when=lambda cfg: BooleanExpression.evaluate(
+            "target={{.+}}-apple-macosx{{15(.[0-9]+)?(.[0-9]+)?}}",
+            cfg.available_features,
+        ),
+    ),
     Feature(
         name="_target-has-llvm-17",
         when=lambda cfg: BooleanExpression.evaluate(
-            "target={{.+}}-apple-macosx{{14.[4-9](.0)?}} || target={{.+}}-apple-macosx{{1[5-9]([.].+)?}}",
+            "_target-has-llvm-18 || target={{.+}}-apple-macosx{{14.[4-9](.0)?}} || target={{.+}}-apple-macosx{{1[5-9]([.].+)?}}",
             cfg.available_features,
         ),
     ),
diff --git a/libcxx/vendor/llvm/default_assertion_handler.in b/libcxx/vendor/llvm/default_assertion_handler.in
index 8f37b9089ece8..e12ccccdaff37 100644
--- a/libcxx/vendor/llvm/default_assertion_handler.in
+++ b/libcxx/vendor/llvm/default_assertion_handler.in
@@ -27,7 +27,7 @@
 // AppleClang shipped a slightly different version of __builtin_verbose_trap from the upstream
 // version before upstream Clang actually got the builtin.
 // TODO: Remove once AppleClang supports the two-arguments version of the builtin.
-#    if defined(_LIBCPP_APPLE_CLANG_VER) && _LIBCPP_APPLE_CLANG_VER < 17000
+#    if defined(_LIBCPP_APPLE_CLANG_VER) && _LIBCPP_APPLE_CLANG_VER < 1700
 #      define _LIBCPP_ASSERTION_HANDLER(message) __builtin_verbose_trap(message)
 #    else
 #      define _LIBCPP_ASSERTION_HANDLER(message) __builtin_verbose_trap("libc++", message)
diff --git a/libunwind/CMakeLists.txt b/libunwind/CMakeLists.txt
index 28d67b0fef92c..bc57057a70529 100644
--- a/libunwind/CMakeLists.txt
+++ b/libunwind/CMakeLists.txt
@@ -31,11 +31,6 @@ include(CMakeDependentOption)
 include(HandleCompilerRT)
 
 # Define options.
-option(LIBUNWIND_BUILD_32_BITS "Build 32 bit multilib libunwind. This option is not supported anymore when building the runtimes. Please specify a full triple instead." ${LLVM_BUILD_32_BITS})
-if (LIBUNWIND_BUILD_32_BITS)
-  message(FATAL_ERROR "LIBUNWIND_BUILD_32_BITS is not supported anymore when building the runtimes, please specify a full triple instead.")
-endif()
-
 option(LIBUNWIND_ENABLE_CET "Build libunwind with CET enabled." OFF)
 option(LIBUNWIND_ENABLE_GCS "Build libunwind with GCS enabled." OFF)
 option(LIBUNWIND_ENABLE_ASSERTIONS "Enable assertions independent of build mode." ON)
@@ -66,6 +61,12 @@ cmake_dependent_option(LIBUNWIND_INSTALL_SHARED_LIBRARY
   "Install the shared libunwind library." ON
   "LIBUNWIND_ENABLE_SHARED;LIBUNWIND_INSTALL_LIBRARY" OFF)
 
+set(LIBUNWIND_LIBRARY_VERSION "1.0" CACHE STRING
+  "Version of libunwind. This will be reflected in the name of the shared library produced.
+   For example, -DLIBUNWIND_LIBRARY_VERSION=x.y will result in the library being named
+   libunwind.x.y.dylib, along with the usual symlinks pointing to that. On Apple platforms,
+   this also controls the linker's 'current_version' property.")
+
 if(MINGW)
   set(LIBUNWIND_DEFAULT_TEST_CONFIG "llvm-libunwind-mingw.cfg.in")
 elseif (LIBUNWIND_ENABLE_SHARED)
diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt
index 780430ba70ba6..125cf4ffe912a 100644
--- a/libunwind/src/CMakeLists.txt
+++ b/libunwind/src/CMakeLists.txt
@@ -105,15 +105,6 @@ if (LIBUNWIND_ENABLE_SHARED AND
           "on C++ ABI library is not supported.")
 endif()
 
-if (APPLE)
-  add_compile_flags("-U__STRICT_ANSI__")
-  add_link_flags("-compatibility_version 1" "-install_name /usr/lib/libunwind.1.dylib")
-
-  if (CMAKE_OSX_DEPLOYMENT_TARGET STREQUAL "10.6")
-    add_link_flags("-current_version ${LIBUNWIND_VERSION}" "/usr/lib/libSystem.B.dylib")
-  endif ()
-endif ()
-
 if (HAIKU)
   add_library_flags_if(LIBUNWIND_HAS_ROOT_LIB root)
 
@@ -170,8 +161,8 @@ if (LIBUNWIND_ENABLE_SHARED)
       LINK_FLAGS "${LIBUNWIND_LINK_FLAGS}"
       LINKER_LANGUAGE C
       OUTPUT_NAME "${LIBUNWIND_SHARED_OUTPUT_NAME}"
-      VERSION "1.0"
-      SOVERSION "1"
+      VERSION     "${LIBUNWIND_LIBRARY_VERSION}"
+      SOVERSION   "1"
   )
 
   list(APPEND LIBUNWIND_BUILD_TARGETS "unwind_shared")
diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index 6510c637ae8fe..c6986681dffe7 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -1100,6 +1100,13 @@ void CHPERedirectionChunk::writeTo(uint8_t *buf) const {
 ImportThunkChunkARM64EC::ImportThunkChunkARM64EC(ImportFile *file)
     : ImportThunkChunk(file->ctx, file->impSym), file(file) {}
 
+size_t ImportThunkChunkARM64EC::getSize() const {
+  if (!extended)
+    return sizeof(importThunkARM64EC);
+  // The last instruction is replaced with an inline range extension thunk.
+  return sizeof(importThunkARM64EC) + sizeof(arm64Thunk) - sizeof(uint32_t);
+}
+
 void ImportThunkChunkARM64EC::writeTo(uint8_t *buf) const {
   memcpy(buf, importThunkARM64EC, sizeof(importThunkARM64EC));
   applyArm64Addr(buf, file->impSym->getRVA(), rva, 12);
@@ -1116,7 +1123,29 @@ void ImportThunkChunkARM64EC::writeTo(uint8_t *buf) const {
   applyArm64Imm(buf + 12, exitThunkRVA & 0xfff, 0);
 
   Defined *helper = cast<Defined>(file->ctx.config.arm64ECIcallHelper);
-  applyArm64Branch26(buf + 16, helper->getRVA() - rva - 16);
+  if (extended) {
+    // Replace last instruction with an inline range extension thunk.
+    memcpy(buf + 16, arm64Thunk, sizeof(arm64Thunk));
+    applyArm64Addr(buf + 16, helper->getRVA(), rva + 16, 12);
+    applyArm64Imm(buf + 20, helper->getRVA() & 0xfff, 0);
+  } else {
+    applyArm64Branch26(buf + 16, helper->getRVA() - rva - 16);
+  }
+}
+
+bool ImportThunkChunkARM64EC::verifyRanges() {
+  if (extended)
+    return true;
+  auto helper = cast<Defined>(file->ctx.config.arm64ECIcallHelper);
+  return isInt<28>(helper->getRVA() - rva - 16);
+}
+
+uint32_t ImportThunkChunkARM64EC::extendRanges() {
+  if (extended || verifyRanges())
+    return 0;
+  extended = true;
+  // The last instruction is replaced with an inline range extension thunk.
+  return sizeof(arm64Thunk) - sizeof(uint32_t);
 }
 
 } // namespace lld::coff
diff --git a/lld/COFF/Chunks.h b/lld/COFF/Chunks.h
index 04a656ae0874e..42284f485e5c0 100644
--- a/lld/COFF/Chunks.h
+++ b/lld/COFF/Chunks.h
@@ -185,6 +185,13 @@ class NonSectionChunk : public Chunk {
   // bytes, so this is used only for logging or debugging.
   virtual StringRef getDebugName() const { return ""; }
 
+  // Verify that chunk relocations are within their ranges.
+  virtual bool verifyRanges() { return true; };
+
+  // If needed, extend the chunk to ensure all relocations are within the
+  // allowed ranges. Return the additional space required for the extension.
+  virtual uint32_t extendRanges() { return 0; };
+
   static bool classof(const Chunk *c) { return c->kind() >= OtherKind; }
 
 protected:
@@ -620,12 +627,15 @@ class ImportThunkChunkARM64 : public ImportThunkChunk {
 class ImportThunkChunkARM64EC : public ImportThunkChunk {
 public:
   explicit ImportThunkChunkARM64EC(ImportFile *file);
-  size_t getSize() const override { return sizeof(importThunkARM64EC); };
+  size_t getSize() const override;
   MachineTypes getMachine() const override { return ARM64EC; }
   void writeTo(uint8_t *buf) const override;
+  bool verifyRanges() override;
+  uint32_t extendRanges() override;
 
   Defined *exitThunk;
   Defined *sym = nullptr;
+  bool extended = false;
 
 private:
   ImportFile *file;
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 7cf723a8cf103..efab7d3e83709 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -473,8 +473,14 @@ bool Writer::createThunks(OutputSection *os, int margin) {
   // elements into it.
   for (size_t i = 0; i != os->chunks.size(); ++i) {
     SectionChunk *sc = dyn_cast<SectionChunk>(os->chunks[i]);
-    if (!sc)
+    if (!sc) {
+      auto chunk = cast<NonSectionChunk>(os->chunks[i]);
+      if (uint32_t size = chunk->extendRanges()) {
+        thunksSize += size;
+        addressesChanged = true;
+      }
       continue;
+    }
     MachineTypes machine = sc->getMachine();
     size_t thunkInsertionSpot = i + 1;
 
@@ -607,8 +613,11 @@ void Writer::createECCodeMap() {
 bool Writer::verifyRanges(const std::vector<Chunk *> chunks) {
   for (Chunk *c : chunks) {
     SectionChunk *sc = dyn_cast<SectionChunk>(c);
-    if (!sc)
+    if (!sc) {
+      if (!cast<NonSectionChunk>(c)->verifyRanges())
+        return false;
       continue;
+    }
     MachineTypes machine = sc->getMachine();
 
     ArrayRef<coff_relocation> relocs = sc->getRelocs();
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index dcdd74ac74f5f..8f34b156c9c4e 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -231,7 +231,7 @@ static std::tuple<ELFKind, uint16_t, uint8_t> parseEmulation(StringRef emul) {
 // Returns slices of MB by parsing MB as an archive file.
 // Each slice consists of a member file in the archive.
 std::vector<std::pair<MemoryBufferRef, uint64_t>> static getArchiveMembers(
-    MemoryBufferRef mb) {
+    Ctx &ctx, MemoryBufferRef mb) {
   std::unique_ptr<Archive> file =
       CHECK(Archive::create(mb),
             mb.getBufferIdentifier() + ": failed to parse archive");
@@ -296,7 +296,7 @@ void LinkerDriver::addFile(StringRef path, bool withLOption) {
     readLinkerScript(ctx, mbref);
     return;
   case file_magic::archive: {
-    auto members = getArchiveMembers(mbref);
+    auto members = getArchiveMembers(ctx, mbref);
     if (inWholeArchive) {
       for (const std::pair<MemoryBufferRef, uint64_t> &p : members) {
         if (isBitcode(p.first))
@@ -632,7 +632,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
 
   // Handle -help
   if (args.hasArg(OPT_help)) {
-    printHelp();
+    printHelp(ctx);
     return;
   }
 
@@ -994,7 +994,7 @@ static void readCallGraph(Ctx &ctx, MemoryBufferRef mb) {
 // true and populates cgProfile and symbolIndices.
 template <class ELFT>
 static bool
-processCallGraphRelocations(SmallVector<uint32_t, 32> &symbolIndices,
+processCallGraphRelocations(Ctx &ctx, SmallVector<uint32_t, 32> &symbolIndices,
                             ArrayRef<typename ELFT::CGProfile> &cgProfile,
                             ObjFile<ELFT> *inputObj) {
   if (inputObj->cgProfileSectionIndex == SHN_UNDEF)
@@ -1046,7 +1046,7 @@ template <class ELFT> static void readCallGraphsFromObjectFiles(Ctx &ctx) {
   ArrayRef<typename ELFT::CGProfile> cgProfile;
   for (auto file : ctx.objectFiles) {
     auto *obj = cast<ObjFile<ELFT>>(file);
-    if (!processCallGraphRelocations(symbolIndices, cgProfile, obj))
+    if (!processCallGraphRelocations(ctx, symbolIndices, cgProfile, obj))
       continue;
 
     if (symbolIndices.size() != cgProfile.size() * 2)
@@ -2378,13 +2378,12 @@ static void replaceCommonSymbols(Ctx &ctx) {
 
 // The section referred to by `s` is considered address-significant. Set the
 // keepUnique flag on the section if appropriate.
-static void markAddrsig(Symbol *s) {
+static void markAddrsig(bool icfSafe, Symbol *s) {
+  // We don't need to keep text sections unique under --icf=all even if they
+  // are address-significant.
   if (auto *d = dyn_cast_or_null<Defined>(s))
-    if (d->section)
-      // We don't need to keep text sections unique under --icf=all even if they
-      // are address-significant.
-      if (ctx.arg.icf == ICFLevel::Safe || !(d->section->flags & SHF_EXECINSTR))
-        d->section->keepUnique = true;
+    if (d->section && (icfSafe || !(d->section->flags & SHF_EXECINSTR)))
+      d->section->keepUnique = true;
 }
 
 // Record sections that define symbols mentioned in --keep-unique <symbol>
@@ -2409,9 +2408,10 @@ static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
 
   // Symbols in the dynsym could be address-significant in other executables
   // or DSOs, so we conservatively mark them as address-significant.
+  bool icfSafe = ctx.arg.icf == ICFLevel::Safe;
   for (Symbol *sym : ctx.symtab->getSymbols())
     if (sym->includeInDynsym())
-      markAddrsig(sym);
+      markAddrsig(icfSafe, sym);
 
   // Visit the address-significance table in each object file and mark each
   // referenced symbol as address-significant.
@@ -2428,14 +2428,14 @@ static void findKeepUniqueSections(Ctx &ctx, opt::InputArgList &args) {
         uint64_t symIndex = decodeULEB128(cur, &size, contents.end(), &err);
         if (err)
           fatal(toString(f) + ": could not decode addrsig section: " + err);
-        markAddrsig(syms[symIndex]);
+        markAddrsig(icfSafe, syms[symIndex]);
         cur += size;
       }
     } else {
       // If an object file does not have an address-significance table,
       // conservatively mark all of its symbols as address-significant.
       for (Symbol *s : syms)
-        markAddrsig(s);
+        markAddrsig(icfSafe, s);
     }
   }
 }
@@ -2497,7 +2497,7 @@ static void readSymbolPartitionSection(Ctx &ctx, InputSectionBase *s) {
   sym->partition = newPart.getNumber();
 }
 
-static void markBuffersAsDontNeed(bool skipLinkedOutput) {
+static void markBuffersAsDontNeed(Ctx &ctx, bool skipLinkedOutput) {
   // With --thinlto-index-only, all buffers are nearly unused from now on
   // (except symbol/section names used by infrequent passes). Mark input file
   // buffers as MADV_DONTNEED so that these pages can be reused by the expensive
@@ -2535,7 +2535,7 @@ void LinkerDriver::compileBitcodeFiles(bool skipLinkedOutput) {
     lto->add(*file);
 
   if (!ctx.bitcodeFiles.empty())
-    markBuffersAsDontNeed(skipLinkedOutput);
+    markBuffersAsDontNeed(ctx, skipLinkedOutput);
 
   for (InputFile *file : lto->compile()) {
     auto *obj = cast<ObjFile<ELFT>>(file);
@@ -2569,7 +2569,8 @@ struct WrappedSymbol {
 // This function instantiates wrapper symbols. At this point, they seem
 // like they are not being used at all, so we explicitly set some flags so
 // that LTO won't eliminate them.
-static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
+static std::vector<WrappedSymbol> addWrappedSymbols(Ctx &ctx,
+                                                    opt::InputArgList &args) {
   std::vector<WrappedSymbol> v;
   DenseSet<StringRef> seen;
 
@@ -2620,7 +2621,7 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
   return v;
 }
 
-static void combineVersionedSymbol(Symbol &sym,
+static void combineVersionedSymbol(Ctx &ctx, Symbol &sym,
                                    DenseMap<Symbol *, Symbol *> &map) {
   const char *suffix1 = sym.getVersionSuffix();
   if (suffix1[0] != '@' || suffix1[1] == '@')
@@ -2687,7 +2688,7 @@ static void redirectSymbols(Ctx &ctx, ArrayRef<WrappedSymbol> wrapped) {
   if (ctx.arg.versionDefinitions.size() > 2)
     for (Symbol *sym : ctx.symtab->getSymbols())
       if (sym->hasVersionSuffix)
-        combineVersionedSymbol(*sym, map);
+        combineVersionedSymbol(ctx, *sym, map);
 
   if (map.empty())
     return;
@@ -2927,7 +2928,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   }
 
   // Archive members defining __wrap symbols may be extracted.
-  std::vector<WrappedSymbol> wrapped = addWrappedSymbols(args);
+  std::vector<WrappedSymbol> wrapped = addWrappedSymbols(ctx, args);
 
   // No more lazy bitcode can be extracted at this point. Do post parse work
   // like checking duplicate symbols.
diff --git a/lld/ELF/Driver.h b/lld/ELF/Driver.h
index 29a2b04af7117..f555e5267e32e 100644
--- a/lld/ELF/Driver.h
+++ b/lld/ELF/Driver.h
@@ -15,6 +15,8 @@
 #include <optional>
 
 namespace lld::elf {
+struct Ctx;
+
 // Parses command line options.
 class ELFOptTable : public llvm::opt::GenericOptTable {
 public:
@@ -30,7 +32,7 @@ enum {
 #undef OPTION
 };
 
-void printHelp();
+void printHelp(Ctx &ctx);
 std::string createResponseFile(const llvm::opt::InputArgList &args);
 
 std::optional<std::string> findFromSearchPaths(StringRef path);
diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp
index f001f2c994e4c..d87f2d85e5a52 100644
--- a/lld/ELF/DriverUtils.cpp
+++ b/lld/ELF/DriverUtils.cpp
@@ -135,7 +135,7 @@ opt::InputArgList ELFOptTable::parse(ArrayRef<const char *> argv) {
   return args;
 }
 
-void elf::printHelp() {
+void elf::printHelp(Ctx &ctx) {
   ELFOptTable().printHelp(
       lld::outs(), (ctx.arg.progName + " [options] file...").str().c_str(),
       "lld", false /*ShowHidden*/, true /*ShowAllAliases*/);
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 56668a3908429..d5d9576c79eb5 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -61,8 +61,6 @@ static lto::Config createConfig(Ctx &ctx) {
   c.Options.FunctionSections = true;
   c.Options.DataSections = true;
 
-  c.Options.BBAddrMap = ctx.arg.ltoBBAddrMap;
-
   // Check if basic block sections must be used.
   // Allowed values for --lto-basic-block-sections are "all", "labels",
   // "<file name specifying basic block ids>", or none.  This is the equivalent
@@ -71,7 +69,8 @@ static lto::Config createConfig(Ctx &ctx) {
     if (ctx.arg.ltoBasicBlockSections == "all") {
       c.Options.BBSections = BasicBlockSection::All;
     } else if (ctx.arg.ltoBasicBlockSections == "labels") {
-      c.Options.BBSections = BasicBlockSection::Labels;
+      c.Options.BBAddrMap = true;
+      c.Options.BBSections = BasicBlockSection::None;
     } else if (ctx.arg.ltoBasicBlockSections == "none") {
       c.Options.BBSections = BasicBlockSection::None;
     } else {
@@ -87,6 +86,8 @@ static lto::Config createConfig(Ctx &ctx) {
     }
   }
 
+  c.Options.BBAddrMap = ctx.arg.ltoBBAddrMap;
+
   c.Options.UniqueBasicBlockSectionNames =
       ctx.arg.ltoUniqueBasicBlockSectionNames;
 
diff --git a/lld/test/COFF/arm64ec-import-range-ext.test b/lld/test/COFF/arm64ec-import-range-ext.test
new file mode 100644
index 0000000000000..701d4c11cc564
--- /dev/null
+++ b/lld/test/COFF/arm64ec-import-range-ext.test
@@ -0,0 +1,39 @@
+REQUIRES: aarch64, x86
+RUN: split-file %s %t.dir && cd %t.dir
+
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows test.s -o test.obj
+RUN: llvm-mc -filetype=obj -triple=arm64ec-windows %S/Inputs/loadconfig-arm64ec.s -o loadconfig-arm64ec.obj
+RUN: llvm-lib -machine:arm64ec -def:test.def -out:test.lib
+
+RUN: lld-link -machine:arm64ec -dll -noentry -out:out.dll loadconfig-arm64ec.obj test.obj test.lib
+
+RUN: llvm-objdump -d out.dll | FileCheck --check-prefix=DISASM %s
+DISASM:      0000000180001000 <.text>:
+DISASM-NEXT: 180001000: 52800000     mov     w0, #0x0                // =0
+DISASM-NEXT: 180001004: d65f03c0     ret
+DISASM-NEXT:                 ...
+DISASM-NEXT: 188001008: b000000b     adrp    x11, 0x188002000
+DISASM-NEXT: 18800100c: f940016b     ldr     x11, [x11]
+DISASM-NEXT: 188001010: f0fbffea     adrp    x10, 0x180000000
+DISASM-NEXT: 188001014: 9100014a     add     x10, x10, #0x0
+DISASM-NEXT: 188001018: 90fc0010     adrp    x16, 0x180001000 <.text>
+DISASM-NEXT: 18800101c: 91000210     add     x16, x16, #0x0
+DISASM-NEXT: 188001020: d61f0200     br      x16
+
+#--- test.s
+    .text
+    .globl __icall_helper_arm64ec
+    .p2align 2, 0x0
+__icall_helper_arm64ec:
+    mov w0, #0
+    ret
+
+    .space 0x8000000
+
+    .data
+    .rva __imp_func
+
+#--- test.def
+NAME test.dll
+EXPORTS
+    func
diff --git a/lldb/docs/use/aarch64-linux.rst b/lldb/docs/use/aarch64-linux.md
similarity index 61%
rename from lldb/docs/use/aarch64-linux.rst
rename to lldb/docs/use/aarch64-linux.md
index c790d9d369a48..803f56d16f981 100644
--- a/lldb/docs/use/aarch64-linux.rst
+++ b/lldb/docs/use/aarch64-linux.md
@@ -1,5 +1,4 @@
-Using LLDB On AArch64 Linux
-===========================
+# Using LLDB On AArch64 Linux
 
 This page explains the details of debugging certain AArch64 extensions using
 LLDB. If something is not mentioned here, it likely works as you would expect.
@@ -8,23 +7,21 @@ This is not a replacement for ptrace and Linux Kernel documentation. This covers
 how LLDB has chosen to use those things and how that effects your experience as
 a user.
 
-Scalable Vector Extension (SVE)
--------------------------------
+## Scalable Vector Extension (SVE)
 
-See `here <https://developer.arm.com/Architectures/Scalable%20Vector%20Extensions>`__
-to learn about the extension and `here <https://kernel.org/doc/html/latest/arch/arm64/sve.html>`__
+See [here](https://developer.arm.com/Architectures/Scalable%20Vector%20Extensions)
+to learn about the extension and [here](https://kernel.org/doc/html/latest/arch/arm64/sve.html)
 for the Linux Kernel's handling of it.
 
 In LLDB you will be able to see the following new registers:
 
-* ``z0-z31`` vector registers, each one has size equal to the vector length.
-* ``p0-p15`` predicate registers, each one containing 1 bit per byte in the vector
+* `z0-z31` vector registers, each one has size equal to the vector length.
+* `p0-p15` predicate registers, each one containing 1 bit per byte in the vector
   length. Making each one vector length / 8 sized.
-* ``ffr`` the first fault register, same size as a predicate register.
-* ``vg``, the vector length in "granules". Each granule is 8 bytes.
-
-.. code-block::
+* `ffr` the first fault register, same size as a predicate register.
+* `vg`, the vector length in "granules". Each granule is 8 bytes.
 
+```
        Scalable Vector Extension Registers:
              vg = 0x0000000000000002
              z0 = {0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 <...> }
@@ -32,19 +29,19 @@ In LLDB you will be able to see the following new registers:
              p0 = {0xff 0xff}
            <...>
             ffr = {0xff 0xff}
+```
 
 The example above has a vector length of 16 bytes. Within LLDB you will always
-see "vg" as in the ``vg`` register, which is 2 in this case (8*2 = 16).
+see "vg" as in the `vg` register, which is 2 in this case (8*2 = 16).
 Elsewhere in kernel code or applications, you may see "vq" which is the vector
 length in quadwords (16 bytes). Where you see "vl", it is in bytes.
 
-While you can count the size of a P or Z register, it is intended that ``vg`` be
+While you can count the size of a P or Z register, it is intended that `vg` be
 used to find the current vector length.
 
-Changing the Vector Length
-..........................
+### Changing the Vector Length
 
-The ``vg`` register can be written during a debug session. Writing the current
+The `vg` register can be written during a debug session. Writing the current
 vector length changes nothing. If you increase the vector length, the registers
 will likely be reset to 0. If you decrease it, LLDB will truncate the Z
 registers but everything else will be reset to 0.
@@ -54,21 +51,20 @@ way the same as it was previously. Whether that is done from within the
 debuggee, or by LLDB. If you need to change the vector length, do so before a
 function's first use of SVE.
 
-Z Register Presentation
-.......................
+### Z Register Presentation
 
 LLDB makes no attempt to predict how SVE Z registers will be used. Since LLDB
 does not know what sort of elements future instructions will interpret the
 register as. It therefore does not change the visualisation of the register
 and always defaults to showing a vector of byte sized elements.
 
-If you know what format you are going to use, give a format option::
-
+If you know what format you are going to use, give a format option:
+```
   (lldb) register read z0 -f uint32_t[]
       z0 = {0x01010101 0x01010101 0x01010101 0x01010101}
+```
 
-FPSIMD and SVE Modes
-....................
+### FPSIMD and SVE Modes
 
 Prior to the debugee's first use of SVE, it is in what the Linux Kernel terms
 SIMD mode. Only the FPU is being used. In this state LLDB will still show the
@@ -82,18 +78,16 @@ You can also trigger this with LLDB by writing to an SVE register. Note that
 there is no way to undo this change from within LLDB. However, the debugee
 itself could do something to end up back in SIMD mode.
 
-Expression evaluation
-.....................
+### Expression evaluation
 
 If you evaluate an expression, all SVE state is saved prior to, and restored
 after the expression has been evaluated. Including the register values and
 vector length.
 
-Scalable Matrix Extension (SME)
--------------------------------
+## Scalable Matrix Extension (SME)
 
-See `here <https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/scalable-matrix-extension-armv9-a-architecture>`__
-to learn about the extension and `here <https://kernel.org/doc/html/latest/arch/arm64/sme.html>`__
+See [here](https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/scalable-matrix-extension-armv9-a-architecture)
+to learn about the extension and [here](https://kernel.org/doc/html/latest/arch/arm64/sme.html)
 for the Linux Kernel's handling of it.
 
 SME adds a "Streaming Mode" to SVE, and this mode has its own vector length
@@ -101,37 +95,38 @@ known as the "Streaming Vector Length".
 
 In LLDB you will see the following new registers:
 
-* ``tpidr2``, an extra per thread pointer reserved for use by the SME ABI.
+* `tpidr2`, an extra per thread pointer reserved for use by the SME ABI.
   This is not scalable, just pointer sized aka 64 bit.
-* ``z0-z31`` streaming SVE registers. These have the same names as the
+* `z0-z31` streaming SVE registers. These have the same names as the
   non-streaming registers and therefore you will only see the active set in
   LLDB. You cannot read or write the inactive mode's registers. Their size
   is the same as the streaming vector length.
-* ``za`` the Array Storage register. The "Matrix" part of "Scalable Matrix
+* `za` the Array Storage register. The "Matrix" part of "Scalable Matrix
   Extension". This is a square made up of rows of length equal to the streaming
   vector length (svl). Meaning that the total size is svl * svl.
-* ``svcr`` the Streaming Vector Control Register. This is actually a pseduo
-  register but it matches the content of the architecturaly defined ``SVCR``.
+* `svcr` the Streaming Vector Control Register. This is actually a pseduo
+  register but it matches the content of the architecturaly defined `SVCR`.
   This is the register you should use to check whether streaming mode and/or
-  ``za`` is active. This register is read only.
-* ``svg`` the streaming vector length in granules. This value is not connected
+  `za` is active. This register is read only.
+* `svg` the streaming vector length in granules. This value is not connected
   to the vector length of non-streaming mode and may change independently. This
   register is read only.
 
-.. note::
-  While in non-streaming mode, the ``vg`` register shows the non-streaming
-  vector length, and the ``svg`` register shows the streaming vector length.
-  When in streaming mode, both ``vg`` and ``svg`` show the streaming mode vector
+```{note}
+  While in non-streaming mode, the `vg` register shows the non-streaming
+  vector length, and the `svg` register shows the streaming vector length.
+  When in streaming mode, both `vg` and `svg` show the streaming mode vector
   length. Therefore it is not possible at this time to read the non-streaming
   vector length within LLDB, while in streaming mode. This is a limitation of
   the LLDB implementation not the architecture, which stores both lengths
   independently.
+```
 
 In the example below, the streaming vector length is 16 bytes and we are in
-streaming mode. Note that bits 0 and 1 of ``svcr`` are set, indicating that we
-are in streaming mode and ZA is active. ``vg`` and ``svg`` report the same value
-as ``vg`` is showing the streaming mode vector length::
-
+streaming mode. Note that bits 0 and 1 of `svcr` are set, indicating that we
+are in streaming mode and ZA is active. `vg` and `svg` report the same value
+as `vg` is showing the streaming mode vector length:
+```
   Scalable Vector Extension Registers:
         vg = 0x0000000000000002
         z0 = {0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 <...> }
@@ -150,11 +145,11 @@ as ``vg`` is showing the streaming mode vector length::
          svg = 0x0000000000000002
         svcr = 0x0000000000000003
           za = {0x00 <...> 0x00}
+```
 
-Changing the Streaming Vector Length
-....................................
+### Changing the Streaming Vector Length
 
-To reduce complexity for LLDB, ``svg`` is read only. This means that you can
+To reduce complexity for LLDB, `svg` is read only. This means that you can
 only change the streaming vector length using LLDB when the debugee is in
 streaming mode.
 
@@ -162,80 +157,75 @@ As for non-streaming SVE, doing so will essentially make the content of the SVE
 registers undefined. It will also disable ZA, which follows what the Linux
 Kernel does.
 
-Visibility of an Inactive ZA Register
-.....................................
+### Visibility of an Inactive ZA Register
 
 LLDB does not handle registers that can come and go at runtime (SVE changes
-size but it does not dissappear). Therefore when ``za`` is not enabled, LLDB
+size but it does not dissappear). Therefore when `za` is not enabled, LLDB
 will return a block of 0s instead. This block will match the expected size of
-``za``::
-
+`za`:
+```
   (lldb) register read za svg svcr
       za = {0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 <...> }
      svg = 0x0000000000000002
     svcr = 0x0000000000000001
+```
 
-Note that ``svcr`` bit 2 is not set, meaning ``za`` is inactive.
+Note that `svcr` bit 2 is not set, meaning `za` is inactive.
 
-If you were to write to ``za`` from LLDB, ``za`` will be made active. There is
+If you were to write to `za` from LLDB, `za` will be made active. There is
 no way from within LLDB to reverse this change. As for changing the vector
-length, the debugee could still do something that would disable ``za`` again.
+length, the debugee could still do something that would disable `za` again.
 
-If you want to know whether ``za`` is active or not, refer to bit 2 of the
-``svcr`` register, otherwise known as ``SVCR.ZA``.
+If you want to know whether `za` is active or not, refer to bit 2 of the
+`svcr` register, otherwise known as `SVCR.ZA`.
 
-ZA Register Presentation
-........................
+### ZA Register Presentation
 
-As for SVE, LLDB does not know how the debugee will use ``za``, and therefore
+As for SVE, LLDB does not know how the debugee will use `za`, and therefore
 does not know how it would be best to display it. At any time any given
 instrucion could interpret its contents as many kinds and sizes of data.
 
-So LLDB will default to showing  ``za`` as one large vector of individual bytes.
+So LLDB will default to showing  `za` as one large vector of individual bytes.
 You can override this with a format option (see the SVE example above).
 
-Expression Evaluation
-.....................
+### Expression Evaluation
 
 The mode (streaming or non-streaming), streaming vector length and ZA state will
 be restored after expression evaluation. On top of all the things saved for SVE
 in general.
 
-Scalable Matrix Extension (SME2)
---------------------------------
+## Scalable Matrix Extension (SME2)
 
 The Scalable Matrix Extension 2 is documented in the same architecture
 specification as SME, and covered by the same kernel documentation page as SME.
 
-SME2 adds 1 new register, ``zt0``. This register is a fixed size 512 bit
+SME2 adds 1 new register, `zt0`. This register is a fixed size 512 bit
 register that is used by new instructions added in SME2. It is shown in LLDB in
 the existing SME register set.
 
-``zt0`` can be active or inactive, as ``za`` can. The same ``SVCR.ZA`` bit
-controls this. An inactive ``zt0`` is shown as 0s, like ``za`` is. Though in
-``zt0``'s case, LLDB does not need to fake the value. Ptrace already returns a
-block of 0s for an inactive ``zt0``.
+`zt0` can be active or inactive, as `za` can. The same `SVCR.ZA` bit
+controls this. An inactive `zt0` is shown as 0s, like `za` is. Though in
+`zt0`'s case, LLDB does not need to fake the value. Ptrace already returns a
+block of 0s for an inactive `zt0`.
 
-Like ``za``, writing to an inactive ``zt0`` will enable it and ``za``. This can
-be done from within LLDB. If the write is instead to ``za``, ``zt0`` becomes
+Like `za`, writing to an inactive `zt0` will enable it and `za`. This can
+be done from within LLDB. If the write is instead to `za`, `zt0` becomes
 active but with a value of all 0s.
 
-Since ``svcr`` is read only, there is no way at this time to deactivate the
+Since `svcr` is read only, there is no way at this time to deactivate the
 registers from within LLDB (though of course a running process can still do
 this).
 
-To check whether ``zt0`` is active, refer to ``SVCR.ZA`` and not to the value of
-``zt0``.
+To check whether `zt0` is active, refer to `SVCR.ZA` and not to the value of
+`zt0`.
 
-ZT0 Register Presentation
-.........................
+### ZT0 Register Presentation
 
-As for ``za``, the meaning of ``zt0`` depends on the instructions used with it,
+As for `za`, the meaning of `zt0` depends on the instructions used with it,
 so LLDB does not attempt to guess this and defaults to showing it as a vector of
 bytes.
 
-Expression Evaluation
-.....................
+### Expression Evaluation
 
-``zt0``'s value and whether it is active or not will be saved prior to
+`zt0`'s value and whether it is active or not will be saved prior to
 expression evaluation and restored afterwards.
\ No newline at end of file
diff --git a/lldb/docs/use/tutorial.rst b/lldb/docs/use/tutorial.rst
index 00e7befdd087a..76e8ac4aeab89 100644
--- a/lldb/docs/use/tutorial.rst
+++ b/lldb/docs/use/tutorial.rst
@@ -536,6 +536,43 @@ This command will run the thread in the current frame until it reaches line 100
 in this frame or stops if it leaves the current frame. This is a pretty close
 equivalent to GDB's ``until`` command.
 
+One other useful thing to note about the lldb stepping commands is that they
+are implemented as a stack of interruptible operations.  Until the operation -
+e.g. step to the next line - is completed, it will remain on the
+stack.  If the step over is interrupted and control returned to you,
+any new stepping commands you issue won't replace the step-over, but instead
+their operations will be pushed onto the stack after the original step over.
+Then each of them will be retired as they are completed, finally returning to the
+original step over operation.
+
+Suppose, for instance, you ``step-over`` a source line with a function call.
+If there is a breakpoint in that function, hitting the breakpoint will interrupt
+the step over.  At that point, you will likely want to examine the state at
+the breakpoint, maybe stepping around in that frame, or stepping into other
+functions, running some expressions, etc.
+
+Because the original step-over has remained on the stack, when you've finished
+your examinations, a simple ``continue`` will resume the original ``step-over``
+operation, and you will arrive at the end of your starting source line in the
+original frame.
+
+This saves you from having to keep track of your original intention, and manually
+issuing the requisite number of ``step-out`` commands to get back to the frame
+you were stepping over.  The stack maintains that information for you.
+
+Hand-called functions using the ``expr`` command are also implemented by
+operations on this same stack.  So if you are calling some code with the ``expr`` command,
+and hit a breakpoint during the evaluation of that code, you can examine
+the state where you stopped, and when you're satisfied,  issue a
+``continue`` to finish the expression evaluation operation and print the function
+result.
+
+You can examine the state of the operations stack using the ``thread plan list``
+command, and if, for instance, you decide you don't actually want that outermost
+next to continue running, you can remove it with the ``thread plan discard``
+command.  If you are interested in following this process in more detail, the
+``lldb step`` logging channel is useful source of information.
+
 A process, by default, will share the LLDB terminal with the inferior process.
 When in this mode, much like when debugging with GDB, when the process is
 running anything you type will go to the ``STDIN`` of the inferior process. To
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index c6b7ce84109c0..8884ef5933ada 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -172,9 +172,9 @@
 WATCHPOINT_CREATED = "Watchpoint created successfully"
 
 
-def CMD_MSG(str):
+def CMD_MSG(command):
     """A generic "Command '%s' did not return successfully" message generator."""
-    return "Command '%s' did not return successfully" % str
+    return f"Command '{command}' did not return successfully"
 
 
 def COMPLETION_MSG(str_before, str_after, completions):
@@ -990,16 +990,14 @@ def runCmd(self, cmd, msg=None, check=True, trace=False, inHistory=False):
                     print("Command '" + cmd + "' failed!", file=sbuf)
 
         if check:
+            if not msg:
+                msg = CMD_MSG(cmd)
             output = ""
             if self.res.GetOutput():
                 output += "\nCommand output:\n" + self.res.GetOutput()
             if self.res.GetError():
                 output += "\nError output:\n" + self.res.GetError()
-            if msg:
-                msg += output
-            if cmd:
-                cmd += output
-            self.assertTrue(self.res.Succeeded(), msg if (msg) else CMD_MSG(cmd))
+            self.assertTrue(self.res.Succeeded(), msg + output)
 
     def HideStdout(self):
         """Hide output to stdout from the user.
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 624848dee6ec3..30242038a5f66 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -594,7 +594,7 @@ void DynamicLoaderDarwin::UpdateSpecialBinariesFromNewImageInfos(
   }
 }
 
-void DynamicLoaderDarwin::UpdateDYLDImageInfoFromNewImageInfo(
+bool DynamicLoaderDarwin::UpdateDYLDImageInfoFromNewImageInfo(
     ImageInfo &image_info) {
   if (image_info.header.filetype == llvm::MachO::MH_DYLINKER) {
     const bool can_create = true;
@@ -605,8 +605,10 @@ void DynamicLoaderDarwin::UpdateDYLDImageInfoFromNewImageInfo(
       target.GetImages().AppendIfNeeded(dyld_sp);
       UpdateImageLoadAddress(dyld_sp.get(), image_info);
       SetDYLDModule(dyld_sp);
+      return true;
     }
   }
+  return false;
 }
 
 std::optional<lldb_private::Address> DynamicLoaderDarwin::GetStartAddress() {
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
index 3613c4c29b178..45c693163f810 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.h
@@ -208,7 +208,7 @@ class DynamicLoaderDarwin : public lldb_private::DynamicLoader {
   UpdateSpecialBinariesFromNewImageInfos(ImageInfo::collection &image_infos);
 
   // if image_info is a dyld binary, call this method
-  void UpdateDYLDImageInfoFromNewImageInfo(ImageInfo &image_info);
+  bool UpdateDYLDImageInfoFromNewImageInfo(ImageInfo &image_info);
 
   // If image_infos contains / may contain executable image, call this method
   // to keep our internal record keeping of the special dyld binary up-to-date.
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
index fe0224483b7c2..debd0f6ee83f4 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
@@ -259,10 +259,13 @@ bool DynamicLoaderMacOSXDYLD::ReadDYLDInfoFromMemoryAndSetNotificationCallback(
       ModuleSP dyld_module_sp;
       if (ParseLoadCommands(data, m_dyld, &m_dyld.file_spec)) {
         if (m_dyld.file_spec) {
-          UpdateDYLDImageInfoFromNewImageInfo(m_dyld);
+          if (!UpdateDYLDImageInfoFromNewImageInfo(m_dyld))
+            return false;
         }
       }
       dyld_module_sp = GetDYLDModule();
+      if (!dyld_module_sp)
+        return false;
 
       Target &target = m_process->GetTarget();
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
index 3e2c208bd2018..fd965d0127a2d 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
@@ -129,8 +129,6 @@ bool ASTResultSynthesizer::SynthesizeFunctionResult(FunctionDecl *FunDecl) {
 
     function_decl->print(os);
 
-    os.flush();
-
     LLDB_LOGF(log, "Untransformed function AST:\n%s", s.c_str());
   }
 
@@ -145,8 +143,6 @@ bool ASTResultSynthesizer::SynthesizeFunctionResult(FunctionDecl *FunDecl) {
 
     function_decl->print(os);
 
-    os.flush();
-
     LLDB_LOGF(log, "Transformed function AST:\n%s", s.c_str());
   }
 
@@ -169,8 +165,6 @@ bool ASTResultSynthesizer::SynthesizeObjCMethodResult(
 
     MethodDecl->print(os);
 
-    os.flush();
-
     LLDB_LOGF(log, "Untransformed method AST:\n%s", s.c_str());
   }
 
@@ -189,8 +183,6 @@ bool ASTResultSynthesizer::SynthesizeObjCMethodResult(
 
     MethodDecl->print(os);
 
-    os.flush();
-
     LLDB_LOGF(log, "Transformed method AST:\n%s", s.c_str());
   }
 
@@ -476,7 +468,6 @@ void ASTResultSynthesizer::CommitPersistentDecls() {
         std::string s;
         llvm::raw_string_ostream ss(s);
         decl->dump(ss);
-        ss.flush();
 
         LLDB_LOGF(log, "Couldn't commit persistent  decl: %s\n", s.c_str());
       }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index adf13ff736adc..630ad7e20ab7e 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -1162,7 +1162,6 @@ void ClangASTImporter::ASTImporterDelegate::ImportDefinitionTo(
         if (NamedDecl *from_named_decl = dyn_cast<clang::NamedDecl>(from)) {
           llvm::raw_string_ostream name_stream(name_string);
           from_named_decl->printName(name_stream);
-          name_stream.flush();
         }
         LLDB_LOG(log_ast,
                  "==== [ClangASTImporter][TUDecl: {0:x}] Imported "
@@ -1292,7 +1291,6 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
       std::string name_string;
       llvm::raw_string_ostream name_stream(name_string);
       from_named_decl->printName(name_stream);
-      name_stream.flush();
 
       LLDB_LOG(
           log,
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 2fe3c0460aa7f..4eeac372a2e65 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -210,7 +210,6 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
     // Render diagnostic message to m_output.
     m_output.clear();
     m_passthrough->HandleDiagnostic(DiagLevel, Info);
-    m_os->flush();
 
     lldb::Severity severity;
     bool make_new_diagnostic = true;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUtil.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUtil.cpp
index 2e0bb318cb507..4cda426e72704 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUtil.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUtil.cpp
@@ -74,7 +74,6 @@ std::string ClangUtil::DumpDecl(const clang::Decl *d) {
   bool deserialize = false;
   d->dump(stream, deserialize);
 
-  stream.flush();
   return result;
 }
 
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 06da83e26a26a..b542e237f023d 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1409,7 +1409,7 @@ void ObjectFileMachO::SanitizeSegmentCommand(
         seg_cmd.cmd == LC_SEGMENT_64 ? "LC_SEGMENT_64" : "LC_SEGMENT";
     GetModule()->ReportWarning(
         "load command {0} {1} has a fileoff + filesize ({2:x16}) that "
-        "extends beyond the end of the file ({4:x16}), the segment will be "
+        "extends beyond the end of the file ({3:x16}), the segment will be "
         "truncated to match",
         cmd_idx, lc_segment_name, seg_cmd.fileoff + seg_cmd.filesize, m_length);
 
diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
index 7c8dba3680938..9f82c935c0e7e 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.cpp
@@ -23,9 +23,33 @@
 #define HWCAP2_AFP (1ULL << 20)
 #define HWCAP2_SME (1ULL << 23)
 #define HWCAP2_EBF16 (1ULL << 32)
+#define HWCAP2_FPMR (1ULL << 48)
 
 using namespace lldb_private;
 
+Arm64RegisterFlagsDetector::Fields
+Arm64RegisterFlagsDetector::DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2) {
+  (void)hwcap;
+
+  if (!(hwcap2 & HWCAP2_FPMR))
+    return {};
+
+  static const FieldEnum fp8_format_enum("fp8_format_enum", {
+                                                                {0, "FP8_E5M2"},
+                                                                {1, "FP8_E4M3"},
+                                                            });
+  return {
+      {"LSCALE2", 32, 37},
+      {"NSCALE", 24, 31},
+      {"LSCALE", 16, 22},
+      {"OSC", 15},
+      {"OSM", 14},
+      {"F8D", 6, 8, &fp8_format_enum},
+      {"F8S2", 3, 5, &fp8_format_enum},
+      {"F8S1", 0, 2, &fp8_format_enum},
+  };
+}
+
 Arm64RegisterFlagsDetector::Fields
 Arm64RegisterFlagsDetector::DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2) {
   (void)hwcap;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
index a5bb38670b9cd..0f3d53d93892b 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterFlagsDetector_arm64.h
@@ -60,6 +60,7 @@ class Arm64RegisterFlagsDetector {
   static Fields DetectFPCRFields(uint64_t hwcap, uint64_t hwcap2);
   static Fields DetectMTECtrlFields(uint64_t hwcap, uint64_t hwcap2);
   static Fields DetectSVCRFields(uint64_t hwcap, uint64_t hwcap2);
+  static Fields DetectFPMRFields(uint64_t hwcap, uint64_t hwcap2);
 
   struct RegisterEntry {
     RegisterEntry(llvm::StringRef name, unsigned size, DetectorFn detector)
@@ -69,12 +70,13 @@ class Arm64RegisterFlagsDetector {
     llvm::StringRef m_name;
     RegisterFlags m_flags;
     DetectorFn m_detector;
-  } m_registers[5] = {
+  } m_registers[6] = {
       RegisterEntry("cpsr", 4, DetectCPSRFields),
       RegisterEntry("fpsr", 4, DetectFPSRFields),
       RegisterEntry("fpcr", 4, DetectFPCRFields),
       RegisterEntry("mte_ctrl", 8, DetectMTECtrlFields),
       RegisterEntry("svcr", 8, DetectSVCRFields),
+      RegisterEntry("fpmr", 8, DetectFPMRFields),
   };
 
   // Becomes true once field detection has been run for all registers.
diff --git a/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp b/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp
index d62eb26ca1a29..ca8e9c078e1f9 100644
--- a/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/DebuggerThread.cpp
@@ -374,7 +374,6 @@ DebuggerThread::HandleCreateProcessEvent(const CREATE_PROCESS_DEBUG_INFO &info,
   std::string thread_name;
   llvm::raw_string_ostream name_stream(thread_name);
   name_stream << "lldb.plugin.process-windows.secondary[" << process_id << "]";
-  name_stream.flush();
   llvm::set_thread_name(thread_name);
 
   // info.hProcess and info.hThread are closed automatically by Windows when
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index d005cf1e3d3c2..e42526c8fd726 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -3702,7 +3702,6 @@ GDBRemoteCommunicationClient::SendTraceStart(const llvm::json::Value &params,
   std::string json_string;
   llvm::raw_string_ostream os(json_string);
   os << params;
-  os.flush();
 
   escaped_packet.PutEscapedBytes(json_string.c_str(), json_string.size());
 
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp
index d4aa90b2c7731..5bd29ae40aa9e 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.cpp
@@ -155,7 +155,6 @@ GDBRemoteCommunicationServer::SendJSONResponse(const json::Value &value) {
   std::string json_string;
   raw_string_ostream os(json_string);
   os << value;
-  os.flush();
   StreamGDBRemote escaped_response;
   escaped_response.PutEscapedBytes(json_string.c_str(), json_string.size());
   return SendPacketNoLock(escaped_response.GetString());
diff --git a/lldb/source/Utility/UUID.cpp b/lldb/source/Utility/UUID.cpp
index 57e3a39d1f8e9..370b8b6848c7e 100644
--- a/lldb/source/Utility/UUID.cpp
+++ b/lldb/source/Utility/UUID.cpp
@@ -56,7 +56,6 @@ std::string UUID::GetAsString(llvm::StringRef separator) const {
 
     os << llvm::format_hex_no_prefix(B.value(), 2, true);
   }
-  os.flush();
 
   return result;
 }
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index ccdb6653cf16f..03cc415924e0b 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -522,3 +522,46 @@ def minidump_deleted_on_save_failure(self):
 
         finally:
             self.assertTrue(self.dbg.DeleteTarget(target))
+
+    def minidump_deterministic_difference(self):
+        """Test that verifies that two minidumps produced are identical."""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+
+            core_styles = [
+                lldb.eSaveCoreStackOnly,
+                lldb.eSaveCoreDirtyOnly,
+                lldb.eSaveCoreFull,
+            ]
+            for style in core_styles:
+                spec_one = lldb.SBFileSpec(self.getBuildArtifact("core.one.dmp"))
+                spec_two = lldb.SBFileSpec(self.getBuildArtifact("core.two.dmp"))
+                options = lldb.SBSaveCoreOptions()
+                options.SetOutputFile(spec_one)
+                options.SetPluginName("minidump")
+                options.SetStyle(style)
+                error = process.SaveCore(options)
+                self.assertTrue(error.Success())
+                options.SetOutputFile(spec_two)
+                error = process.SaveCore(options)
+                self.assertTrue(error.Success())
+
+                file_one = None
+                file_two = None
+                with open(spec_one.GetFileName(), mode="rb") as file:
+                    file_one = file.read()
+                with open(spec_two.GetFileName(), mode="rb") as file:
+                    file_two = file.read()
+                self.assertEqual(file_one, file_two)
+                self.assertTrue(os.unlink(spec_one.GetFileName()))
+                self.assertTrue(os.unlink(spec_two.GetFileName()))
+
+        finally:
+            self.assertTrue(self.dbg.DeleteTarget(target))
diff --git a/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py b/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
index 5a3b8f501095e..d022c8eb3d6cc 100644
--- a/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
+++ b/lldb/test/API/linux/aarch64/fpmr/TestAArch64LinuxFPMR.py
@@ -45,6 +45,11 @@ def test_fpmr_register(self):
             substrs=["Floating Point Mode Register", f"fpmr = {expected_fpmr:#018x}"],
         )
 
+        if self.hasXMLSupport():
+            self.expect(
+                "register read fpmr", substrs=["LSCALE2 = 42", "F8S1 = FP8_E4M3 | 0x4"]
+            )
+
         # Write a value for the program to find. Same fields but with bit values
         # inverted.
         new_fpmr = (0b010101 << 32) | 0b010
diff --git a/lldb/unittests/Process/minidump/MinidumpParserTest.cpp b/lldb/unittests/Process/minidump/MinidumpParserTest.cpp
index c7547ba261c7f..a6d015e79a7ef 100644
--- a/lldb/unittests/Process/minidump/MinidumpParserTest.cpp
+++ b/lldb/unittests/Process/minidump/MinidumpParserTest.cpp
@@ -59,7 +59,6 @@ class MinidumpParserTest : public testing::Test {
       return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                      "convertYAML() failed");
 
-    os.flush();
     auto data_buffer_sp =
         std::make_shared<DataBufferHeap>(data.data(), data.size());
     auto expected_parser = MinidumpParser::Create(std::move(data_buffer_sp));
@@ -85,7 +84,6 @@ TEST_F(MinidumpParserTest, InvalidMinidump) {
   )");
 
   ASSERT_TRUE(llvm::yaml::convertYAML(YIn, os, [](const llvm::Twine &Msg){}));
-  os.flush();
   auto data_buffer_sp = std::make_shared<DataBufferHeap>(
       duplicate_streams.data(), duplicate_streams.size());
   ASSERT_THAT_EXPECTED(MinidumpParser::Create(data_buffer_sp), llvm::Failed());
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 330db65e85cab..0044c38f566a7 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -161,12 +161,6 @@ foreach(proj IN LISTS LLVM_ENABLE_RUNTIMES)
   endif()
 endforeach()
 
-foreach(proj IN LISTS LLVM_ENABLE_RUNTIMES)
-  if ("${proj}" IN_LIST LLVM_ENABLE_PROJECTS)
-    message(FATAL_ERROR "Runtime project \"${proj}\" found in LLVM_ENABLE_PROJECTS and LLVM_ENABLE_RUNTIMES. It must only appear in one of them and that one should almost always be LLVM_ENABLE_RUNTIMES.")
-  endif()
-endforeach()
-
 # Set a shorthand option to enable the GPU build of the 'libc' project.
 option(LIBC_GPU_BUILD "Enable the 'libc' project targeting the GPU" OFF)
 if(LIBC_GPU_BUILD)
@@ -204,6 +198,16 @@ if(NEED_LIBC_HDRGEN)
     list(APPEND LLVM_ENABLE_PROJECTS "libc")
   endif()
 endif()
+
+foreach(proj IN LISTS LLVM_ENABLE_RUNTIMES)
+  if("${proj}" IN_LIST LLVM_ENABLE_PROJECTS)
+    # The 'libc' project bootstraps a few executables via the project build and
+    # should not emit an error currently.
+    if(NOT (NEED_LIBC_HDRGEN AND "${proj}" STREQUAL "libc"))
+      message(FATAL_ERROR "Runtime project \"${proj}\" found in LLVM_ENABLE_PROJECTS and LLVM_ENABLE_RUNTIMES. It must only appear in one of them and that one should almost always be LLVM_ENABLE_RUNTIMES.")
+    endif()
+  endif()
+endforeach()
 unset(NEED_LIBC_HDRGEN)
 
 # LLVM_ENABLE_PROJECTS_USED is `ON` if the user has ever used the
diff --git a/llvm/benchmarks/SandboxIRBench.cpp b/llvm/benchmarks/SandboxIRBench.cpp
index d4601d5f53d07..8f7ab1a376899 100644
--- a/llvm/benchmarks/SandboxIRBench.cpp
+++ b/llvm/benchmarks/SandboxIRBench.cpp
@@ -18,7 +18,8 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Module.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Module.h"
 #include "llvm/Support/SourceMgr.h"
 #include <memory>
 #include <sstream>
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index ed13a82905b4e..e17e2169cd880 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -324,6 +324,12 @@ function(append value)
   endforeach(variable)
 endfunction()
 
+function(prepend value)
+  foreach(variable ${ARGN})
+    set(${variable} "${value} ${${variable}}" PARENT_SCOPE)
+  endforeach(variable)
+endfunction()
+
 function(append_if condition value)
   if (${condition})
     foreach(variable ${ARGN})
@@ -1196,7 +1202,7 @@ if (CLANG_CL AND (LLVM_BUILD_INSTRUMENTED OR LLVM_USE_SANITIZER))
   endif()
   file(TO_CMAKE_PATH "${clang_compiler_rt_file}" clang_compiler_rt_file)
   get_filename_component(clang_runtime_dir "${clang_compiler_rt_file}" DIRECTORY)
-  append("/libpath:\"${clang_runtime_dir}\""
+  prepend("/libpath:\"${clang_runtime_dir}\""
     CMAKE_EXE_LINKER_FLAGS
     CMAKE_MODULE_LINKER_FLAGS
     CMAKE_SHARED_LINKER_FLAGS)
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index 7f8def756c696..ab9f583e96ec6 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -272,7 +272,7 @@ OPTIONS
   When printing a PC-relative global symbol reference, print it as an offset from the leading symbol.
 
   When a bb-address-map section is present (i.e., the object file is built with
-  ``-fbasic-block-sections=labels``), labels are retrieved from that section
+  ``-fbasic-block-address-map``), labels are retrieved from that section
   instead. If a pgo-analysis-map is present alongside the bb-address-map, any
   available analyses are printed after the relevant block label. By default,
   any analysis with a special representation (i.e. BlockFrequency,
diff --git a/llvm/docs/Extensions.rst b/llvm/docs/Extensions.rst
index abc34bc3202c0..ea267842cdc35 100644
--- a/llvm/docs/Extensions.rst
+++ b/llvm/docs/Extensions.rst
@@ -401,7 +401,7 @@ the symbol that belongs to the partition. It may be constructed as follows:
 This section stores the binary address of basic blocks along with other related
 metadata. This information can be used to map binary profiles (like perf
 profiles) directly to machine basic blocks.
-This section is emitted with ``-basic-block-sections=labels`` and will contain
+This section is emitted with ``-basic-block-address-map`` and will contain
 a BB address map table for every function.
 
 The ``SHT_LLVM_BB_ADDR_MAP`` type provides backward compatibility to allow
diff --git a/llvm/docs/GlobalISel/GenericOpcode.rst b/llvm/docs/GlobalISel/GenericOpcode.rst
index c42adc10b10a2..1c4e00b956bc4 100644
--- a/llvm/docs/GlobalISel/GenericOpcode.rst
+++ b/llvm/docs/GlobalISel/GenericOpcode.rst
@@ -633,8 +633,8 @@ G_FCEIL, G_FSQRT, G_FFLOOR, G_FRINT, G_FNEARBYINT
 
 These correspond to the standard C functions of the same name.
 
-G_FCOS, G_FSIN, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FCOSH, G_FSINH, G_FTANH
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+G_FCOS, G_FSIN, G_FTAN, G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH, G_FTANH
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 These correspond to the standard C trigonometry functions of the same name.
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 3b905c2788128..3f39d58b322a4 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -12658,10 +12658,67 @@ This instruction requires several arguments:
       the return value of the callee is returned to the caller's caller, even
       if a void return type is in use.
 
-   Both markers imply that the callee does not access allocas from the caller.
-   The ``tail`` marker additionally implies that the callee does not access
-   varargs from the caller. Calls marked ``musttail`` must obey the following
-   additional  rules:
+   Both markers imply that the callee does not access allocas, va_args, or
+   byval arguments from the caller. As an exception to that, an alloca or byval
+   argument may be passed to the callee as a byval argument, which can be
+   dereferenced inside the callee. For example:
+
+.. code-block:: llvm
+
+      declare void @take_byval(ptr byval(i64))
+      declare void @take_ptr(ptr)
+
+      ; Invalid (assuming @take_ptr dereferences the pointer), because %local
+      ; may be de-allocated before the call to @take_ptr.
+      define void @invalid_alloca() {
+      entry:
+        %local = alloca i64
+        tail call void @take_ptr(ptr %local)
+        ret void
+      }
+
+      ; Valid, the byval attribute causes the memory allocated by %local to be
+      ; copied into @take_byval's stack frame.
+      define void @byval_alloca() {
+      entry:
+        %local = alloca i64
+        tail call void @take_byval(ptr byval(i64) %local)
+        ret void
+      }
+
+      ; Invalid, because @use_global_va_list uses the variadic arguments from
+      ; @invalid_va_list.
+      %struct.va_list = type { ptr }
+      @va_list = external global %struct.va_list
+      define void @use_global_va_list() {
+      entry:
+        %arg = va_arg ptr @va_list, i64
+        ret void
+      }
+      define void @invalid_va_list(i32 %a, ...) {
+      entry:
+        call void @llvm.va_start.p0(ptr @va_list)
+        tail call void @use_global_va_list()
+        ret void
+      }
+
+      ; Valid, byval argument forwarded to tail call as another byval argument.
+      define void @forward_byval(ptr byval(i64) %x) {
+      entry:
+        tail call void @take_byval(ptr byval(i64) %x)
+        ret void
+      }
+
+      ; Invalid (assuming @take_ptr dereferences the pointer), byval argument
+      ; passed to tail callee as non-byval ptr.
+      define void @invalid_byval(ptr byval(i64) %x) {
+      entry:
+        tail call void @take_ptr(ptr %x)
+        ret void
+      }
+
+
+   Calls marked ``musttail`` must obey the following additional rules:
 
    - The call must immediately precede a :ref:`ret <i_ret>` instruction,
      or a pointer bitcast followed by a ret instruction.
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
new file mode 100644
index 0000000000000..f44d636a20337
--- /dev/null
+++ b/llvm/docs/ReleaseNotes.md
@@ -0,0 +1,281 @@
+<!-- This document is written in Markdown and uses extra directives provided by
+MyST (https://myst-parser.readthedocs.io/en/latest/). -->
+
+LLVM {{env.config.release}} Release Notes
+=========================================
+
+```{contents}
+```
+
+````{only} PreRelease
+```{warning} These are in-progress notes for the upcoming LLVM {{env.config.release}}
+             release. Release notes for previous releases can be found on
+             [the Download Page](https://releases.llvm.org/download.html).
+```
+````
+
+Introduction
+============
+
+This document contains the release notes for the LLVM Compiler Infrastructure,
+release {{env.config.release}}.  Here we describe the status of LLVM, including
+major improvements from the previous release, improvements in various subprojects
+of LLVM, and some of the current users of the code.  All LLVM releases may be
+downloaded from the [LLVM releases web site](https://llvm.org/releases/).
+
+For more information about LLVM, including information about the latest
+release, please check out the [main LLVM web site](https://llvm.org/).  If you
+have questions or comments, the [Discourse forums](https://discourse.llvm.org)
+is a good place to ask them.
+
+Note that if you are reading this file from a Git checkout or the main
+LLVM web page, this document applies to the *next* release, not the current
+one.  To see the release notes for a specific release, please see the
+[releases page](https://llvm.org/releases/).
+
+Non-comprehensive list of changes in this release
+=================================================
+
+<!-- For small 1-3 sentence descriptions, just add an entry at the end of
+this list. If your description won't fit comfortably in one bullet
+point (e.g. maybe you would like to give an example of the
+functionality, or simply have a lot to talk about), see the comment below
+for adding a new subsection. -->
+
+* ...
+
+<!-- If you would like to document a larger change, then you can add a
+subsection about it right here. You can copy the following boilerplate:
+
+Special New Feature
+-------------------
+
+Makes programs 10x faster by doing Special New Thing.
+-->
+
+Changes to the LLVM IR
+----------------------
+
+* The `x86_mmx` IR type has been removed. It will be translated to
+  the standard vector type `<1 x i64>` in bitcode upgrade.
+* Renamed `llvm.experimental.stepvector` intrinsic to `llvm.stepvector`.
+
+* Added `usub_cond` and `usub_sat` operations to `atomicrmw`.
+
+* Remove the following intrinsics which can be replaced with a `bitcast`:
+
+  * `llvm.nvvm.bitcast.f2i`
+  * `llvm.nvvm.bitcast.i2f`
+  * `llvm.nvvm.bitcast.d2ll`
+  * `llvm.nvvm.bitcast.ll2d`
+
+* Remove the following intrinsics which can be replaced with a funnel-shift:
+
+  * `llvm.nvvm.rotate.b32`
+  * `llvm.nvvm.rotate.right.b64`
+  * `llvm.nvvm.rotate.b64`
+
+* Remove the following intrinsics which can be replaced with an
+  `addrspacecast`:
+
+  * `llvm.nvvm.ptr.gen.to.global`
+  * `llvm.nvvm.ptr.gen.to.shared`
+  * `llvm.nvvm.ptr.gen.to.constant`
+  * `llvm.nvvm.ptr.gen.to.local`
+  * `llvm.nvvm.ptr.global.to.gen`
+  * `llvm.nvvm.ptr.shared.to.gen`
+  * `llvm.nvvm.ptr.constant.to.gen`
+  * `llvm.nvvm.ptr.local.to.gen`
+
+Changes to LLVM infrastructure
+------------------------------
+
+Changes to building LLVM
+------------------------
+
+Changes to TableGen
+-------------------
+
+Changes to Interprocedural Optimizations
+----------------------------------------
+
+Changes to the AArch64 Backend
+------------------------------
+
+* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
+  the required alignment space with a sequence of `0x0` bytes (the requested
+  fill value) rather than NOPs.
+
+Changes to the AMDGPU Backend
+-----------------------------
+
+* Removed `llvm.amdgcn.flat.atomic.fadd` and
+  `llvm.amdgcn.global.atomic.fadd` intrinsics. Users should use the
+  {ref}`atomicrmw <i_atomicrmw>` instruction with `fadd` and
+  addrspace(0) or addrspace(1) instead.
+
+Changes to the ARM Backend
+--------------------------
+
+* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
+  the required alignment space with a sequence of `0x0` bytes (the requested
+  fill value) rather than NOPs.
+
+Changes to the AVR Backend
+--------------------------
+
+Changes to the DirectX Backend
+------------------------------
+
+Changes to the Hexagon Backend
+------------------------------
+
+Changes to the LoongArch Backend
+--------------------------------
+
+Changes to the MIPS Backend
+---------------------------
+
+Changes to the PowerPC Backend
+------------------------------
+
+Changes to the RISC-V Backend
+-----------------------------
+
+* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
+  the required alignment space with a sequence of `0x0` bytes (the requested
+  fill value) rather than NOPs.
+* Added Syntacore SCR4 and SCR5 CPUs: `-mcpu=syntacore-scr4/5-rv32/64`
+* `-mcpu=sifive-p470` was added.
+* Added Hazard3 CPU as taped out for RP2350: `-mcpu=rp2350-hazard3` (32-bit
+  only).
+* Fixed length vector support using RVV instructions now requires VLEN>=64. This
+  means Zve32x and Zve32f will also require Zvl64b. The prior support was
+  largely untested.
+* The `Zvbc32e` and `Zvkgs` extensions are now supported experimentally.
+* Added `Smctr` and `Ssctr` extensions.
+* `-mcpu=syntacore-scr7` was added.
+* The `Zacas` extension is no longer marked as experimental.
+
+Changes to the WebAssembly Backend
+----------------------------------
+
+Changes to the Windows Target
+-----------------------------
+
+Changes to the X86 Backend
+--------------------------
+
+* `.balign N, 0x90`, `.p2align N, 0x90`, and `.align N, 0x90` in code sections
+  now fill the required alignment space with repeating `0x90` bytes, rather than
+  using optimised NOP filling. Optimised NOP filling fills the space with NOP
+  instructions of various widths, not just those that use the `0x90` byte
+  encoding. To use optimised NOP filling in a code section, leave off the
+  "fillval" argument, i.e. `.balign N`, `.p2align N` or `.align N` respectively.
+
+* Due to the removal of the `x86_mmx` IR type, functions with
+  `x86_mmx` arguments or return values will use a different,
+  incompatible, calling convention ABI. Such functions are not
+  generally seen in the wild (Clang never generates them!), so this is
+  not expected to result in real-world compatibility problems.
+
+* Support ISA of `AVX10.2-256` and `AVX10.2-512`.
+
+Changes to the OCaml bindings
+-----------------------------
+
+Changes to the Python bindings
+------------------------------
+
+Changes to the C API
+--------------------
+
+* The following symbols are deleted due to the removal of the `x86_mmx` IR type:
+
+  * `LLVMX86_MMXTypeKind`
+  * `LLVMX86MMXTypeInContext`
+  * `LLVMX86MMXType`
+
+ * The following functions are added to further support non-null-terminated strings:
+
+  * `LLVMGetNamedFunctionWithLength`
+  * `LLVMGetNamedGlobalWithLength`
+
+* The following functions are added to access the `LLVMContextRef` associated
+   with `LLVMValueRef` and `LLVMBuilderRef` objects:
+
+  * `LLVMGetValueContext`
+  * `LLVMGetBuilderContext`
+
+* The new pass manager can now be invoked with a custom alias analysis pipeline, using
+  the `LLVMPassBuilderOptionsSetAAPipeline` function.
+
+* It is now also possible to run the new pass manager on a single function, by calling
+  `LLVMRunPassesOnFunction` instead of `LLVMRunPasses`.
+
+* Support for creating instructions with custom synchronization scopes has been added:
+
+  * `LLVMGetSyncScopeID` to map a synchronization scope name to an ID.
+  * `LLVMBuildFenceSyncScope`, `LLVMBuildAtomicRMWSyncScope` and
+    `LLVMBuildAtomicCmpXchgSyncScope` versions of the existing builder functions
+    with an additional synchronization scope ID parameter.
+  * `LLVMGetAtomicSyncScopeID` and `LLVMSetAtomicSyncScopeID` to get and set the
+    synchronization scope of any atomic instruction.
+  * `LLVMIsAtomic` to check if an instruction is atomic, for use with the above functions.
+    Because of backwards compatibility, `LLVMIsAtomicSingleThread` and
+    `LLVMSetAtomicSingleThread` continue to work with any instruction type.
+
+* The `LLVMSetPersonalityFn` and `LLVMSetInitializer` APIs now support clearing the
+  personality function and initializer respectively by passing a null pointer.
+
+* The following functions are added to allow iterating over debug records attached to
+  instructions:
+
+  * `LLVMGetFirstDbgRecord`
+  * `LLVMGetLastDbgRecord`
+  * `LLVMGetNextDbgRecord`
+  * `LLVMGetPreviousDbgRecord`
+
+* Added `LLVMAtomicRMWBinOpUSubCond` and `LLVMAtomicRMWBinOpUSubSat` to `LLVMAtomicRMWBinOp` enum for AtomicRMW instructions.
+
+Changes to the CodeGen infrastructure
+-------------------------------------
+
+Changes to the Metadata Info
+---------------------------------
+
+Changes to the Debug Info
+---------------------------------
+
+Changes to the LLVM tools
+---------------------------------
+
+Changes to LLDB
+---------------------------------
+
+Changes to BOLT
+---------------------------------
+
+Changes to Sanitizers
+---------------------
+
+Other Changes
+-------------
+
+External Open Source Projects Using LLVM {{env.config.release}}
+===============================================================
+
+* A project...
+
+Additional Information
+======================
+
+A wide variety of additional information is available on the
+[LLVM web page](https://llvm.org/), in particular in the
+[documentation](https://llvm.org/docs/) section.  The web page also contains
+versions of the API documentation which is up-to-date with the Git version of
+the source code.  You can access versions of these documents specific to this
+release by going into the `llvm/docs/` directory in the LLVM tree.
+
+If you have any questions or comments about LLVM, please feel free to contact
+us via the [Discourse forums](https://discourse.llvm.org).
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
deleted file mode 100644
index 05f5bd65fc5f6..0000000000000
--- a/llvm/docs/ReleaseNotes.rst
+++ /dev/null
@@ -1,257 +0,0 @@
-============================
-LLVM |release| Release Notes
-============================
-
-.. contents::
-    :local:
-
-.. only:: PreRelease
-
-  .. warning::
-     These are in-progress notes for the upcoming LLVM |version| release.
-     Release notes for previous releases can be found on
-     `the Download Page <https://releases.llvm.org/download.html>`_.
-
-
-Introduction
-============
-
-This document contains the release notes for the LLVM Compiler Infrastructure,
-release |release|.  Here we describe the status of LLVM, including major improvements
-from the previous release, improvements in various subprojects of LLVM, and
-some of the current users of the code.  All LLVM releases may be downloaded
-from the `LLVM releases web site <https://llvm.org/releases/>`_.
-
-For more information about LLVM, including information about the latest
-release, please check out the `main LLVM web site <https://llvm.org/>`_.  If you
-have questions or comments, the `Discourse forums
-<https://discourse.llvm.org>`_ is a good place to ask
-them.
-
-Note that if you are reading this file from a Git checkout or the main
-LLVM web page, this document applies to the *next* release, not the current
-one.  To see the release notes for a specific release, please see the `releases
-page <https://llvm.org/releases/>`_.
-
-Non-comprehensive list of changes in this release
-=================================================
-.. NOTE
-   For small 1-3 sentence descriptions, just add an entry at the end of
-   this list. If your description won't fit comfortably in one bullet
-   point (e.g. maybe you would like to give an example of the
-   functionality, or simply have a lot to talk about), see the `NOTE` below
-   for adding a new subsection.
-
-* ...
-
-Update on required toolchains to build LLVM
--------------------------------------------
-
-Changes to the LLVM IR
-----------------------
-
-* The ``x86_mmx`` IR type has been removed. It will be translated to
-  the standard vector type ``<1 x i64>`` in bitcode upgrade.
-* Renamed ``llvm.experimental.stepvector`` intrinsic to ``llvm.stepvector``.
-
-* Added ``usub_cond`` and ``usub_sat`` operations to ``atomicrmw``.
-
-* Remove the following intrinsics which can be replaced with a ``bitcast``:
-
-  * ``llvm.nvvm.bitcast.f2i``
-  * ``llvm.nvvm.bitcast.i2f``
-  * ``llvm.nvvm.bitcast.d2ll``
-  * ``llvm.nvvm.bitcast.ll2d``
-
-Changes to LLVM infrastructure
-------------------------------
-
-Changes to building LLVM
-------------------------
-
-Changes to TableGen
--------------------
-
-Changes to Interprocedural Optimizations
-----------------------------------------
-
-Changes to the AArch64 Backend
-------------------------------
-
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-
-Changes to the AMDGPU Backend
------------------------------
-
-* Removed ``llvm.amdgcn.flat.atomic.fadd`` and
-  ``llvm.amdgcn.global.atomic.fadd`` intrinsics. Users should use the
-  :ref:`atomicrmw <i_atomicrmw>` instruction with `fadd` and
-  addrspace(0) or addrspace(1) instead.
-
-Changes to the ARM Backend
---------------------------
-
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-
-Changes to the AVR Backend
---------------------------
-
-Changes to the DirectX Backend
-------------------------------
-
-Changes to the Hexagon Backend
-------------------------------
-
-Changes to the LoongArch Backend
---------------------------------
-
-Changes to the MIPS Backend
----------------------------
-
-Changes to the PowerPC Backend
-------------------------------
-
-Changes to the RISC-V Backend
------------------------------
-
-* `.balign N, 0`, `.p2align N, 0`, `.align N, 0` in code sections will now fill
-  the required alignment space with a sequence of `0x0` bytes (the requested
-  fill value) rather than NOPs.
-* Added Syntacore SCR4 and SCR5 CPUs: ``-mcpu=syntacore-scr4/5-rv32/64``
-* ``-mcpu=sifive-p470`` was added.
-* Added Hazard3 CPU as taped out for RP2350: ``-mcpu=rp2350-hazard3`` (32-bit
-  only).
-* Fixed length vector support using RVV instructions now requires VLEN>=64. This
-  means Zve32x and Zve32f will also require Zvl64b. The prior support was
-  largely untested.
-* The ``Zvbc32e`` and ``Zvkgs`` extensions are now supported experimentally.
-* Added ``Smctr`` and ``Ssctr`` extensions.
-* ``-mcpu=syntacore-scr7`` was added.
-* The ``Zacas`` extension is no longer marked as experimental.
-
-Changes to the WebAssembly Backend
-----------------------------------
-
-Changes to the Windows Target
------------------------------
-
-Changes to the X86 Backend
---------------------------
-
-* `.balign N, 0x90`, `.p2align N, 0x90`, and `.align N, 0x90` in code sections
-  now fill the required alignment space with repeating `0x90` bytes, rather than
-  using optimised NOP filling. Optimised NOP filling fills the space with NOP
-  instructions of various widths, not just those that use the `0x90` byte
-  encoding. To use optimised NOP filling in a code section, leave off the
-  "fillval" argument, i.e. `.balign N`, `.p2align N` or `.align N` respectively.
-
-* Due to the removal of the ``x86_mmx`` IR type, functions with
-  ``x86_mmx`` arguments or return values will use a different,
-  incompatible, calling convention ABI. Such functions are not
-  generally seen in the wild (Clang never generates them!), so this is
-  not expected to result in real-world compatibility problems.
-
-* Support ISA of ``AVX10.2-256`` and ``AVX10.2-512``.
-
-Changes to the OCaml bindings
------------------------------
-
-Changes to the Python bindings
-------------------------------
-
-Changes to the C API
---------------------
-
-* The following symbols are deleted due to the removal of the ``x86_mmx`` IR type:
-
-  * ``LLVMX86_MMXTypeKind``
-  * ``LLVMX86MMXTypeInContext``
-  * ``LLVMX86MMXType``
-
- * The following functions are added to further support non-null-terminated strings:
-
-  * ``LLVMGetNamedFunctionWithLength``
-  * ``LLVMGetNamedGlobalWithLength``
-
-* The following functions are added to access the ``LLVMContextRef`` associated
-   with ``LLVMValueRef`` and ``LLVMBuilderRef`` objects:
-
-  * ``LLVMGetValueContext``
-  * ``LLVMGetBuilderContext``
-
-* The new pass manager can now be invoked with a custom alias analysis pipeline, using
-  the ``LLVMPassBuilderOptionsSetAAPipeline`` function.
-
-* It is now also possible to run the new pass manager on a single function, by calling
-  ``LLVMRunPassesOnFunction`` instead of ``LLVMRunPasses``.
-
-* Support for creating instructions with custom synchronization scopes has been added:
-
-  * ``LLVMGetSyncScopeID`` to map a synchronization scope name to an ID.
-  * ``LLVMBuildFenceSyncScope``, ``LLVMBuildAtomicRMWSyncScope`` and
-    ``LLVMBuildAtomicCmpXchgSyncScope`` versions of the existing builder functions
-    with an additional synchronization scope ID parameter.
-  * ``LLVMGetAtomicSyncScopeID`` and ``LLVMSetAtomicSyncScopeID`` to get and set the
-    synchronization scope of any atomic instruction.
-  * ``LLVMIsAtomic`` to check if an instruction is atomic, for use with the above functions.
-    Because of backwards compatibility, ``LLVMIsAtomicSingleThread`` and
-    ``LLVMSetAtomicSingleThread`` continue to work with any instruction type.
-
-* The `LLVMSetPersonalityFn` and `LLVMSetInitializer` APIs now support clearing the
-  personality function and initializer respectively by passing a null pointer.
-
-* The following functions are added to allow iterating over debug records attached to
-  instructions:
-
-  * ``LLVMGetFirstDbgRecord``
-  * ``LLVMGetLastDbgRecord``
-  * ``LLVMGetNextDbgRecord``
-  * ``LLVMGetPreviousDbgRecord``
-
-* Added ``LLVMAtomicRMWBinOpUSubCond`` and ``LLVMAtomicRMWBinOpUSubSat`` to ``LLVMAtomicRMWBinOp`` enum for AtomicRMW instructions.
-
-Changes to the CodeGen infrastructure
--------------------------------------
-
-Changes to the Metadata Info
----------------------------------
-
-Changes to the Debug Info
----------------------------------
-
-Changes to the LLVM tools
----------------------------------
-
-Changes to LLDB
----------------------------------
-
-Changes to BOLT
----------------------------------
-
-Changes to Sanitizers
----------------------
-
-Other Changes
--------------
-
-External Open Source Projects Using LLVM 19
-===========================================
-
-* A project...
-
-Additional Information
-======================
-
-A wide variety of additional information is available on the `LLVM web page
-<https://llvm.org/>`_, in particular in the `documentation
-<https://llvm.org/docs/>`_ section.  The web page also contains versions of the
-API documentation which is up-to-date with the Git version of the source
-code.  You can access versions of these documents specific to this release by
-going into the ``llvm/docs/`` directory in the LLVM tree.
-
-If you have any questions or comments about LLVM, please feel free to contact
-us via the `Discourse forums <https://discourse.llvm.org>`_.
diff --git a/llvm/docs/SPIRVUsage.rst b/llvm/docs/SPIRVUsage.rst
index 0f0b21fb23770..bb12b05246afb 100644
--- a/llvm/docs/SPIRVUsage.rst
+++ b/llvm/docs/SPIRVUsage.rst
@@ -275,6 +275,10 @@ SPIR-V backend, along with their descriptions and argument details.
      - None
      - `[Type, Vararg]`
      - Assigns names to types or values, enhancing readability and debuggability of SPIR-V code. Not emitted directly but used for metadata enrichment.
+   * - `int_spv_value_md`
+     - None
+     - `[Metadata]`
+     - Assigns a set of attributes (such as name and data type) to a value that is the argument of the associated `llvm.fake.use` intrinsic call. The latter is used as a mean to map virtual registers created by IRTranslator to the original value.
    * - `int_spv_assign_decoration`
      - None
      - `[Type, Metadata]`
diff --git a/llvm/docs/Security.rst b/llvm/docs/Security.rst
index 2b5b5139858e7..67b6ebb4b04d9 100644
--- a/llvm/docs/Security.rst
+++ b/llvm/docs/Security.rst
@@ -37,7 +37,6 @@ meet the criteria for inclusion below. The list is in the format
 username for an individual isn't available, the brackets will be empty.
 
 * Ahmed Bougacha (Apple) [@ahmedbougacha]
-* Andy Kaylor (Intel) [@andykaylor]
 * Artur Pilipenko (Azul Systems Inc) []
 * Boovaragavan Dasarathan (Nvidia) [@mrragava]
 * Dimitry Andric (individual; FreeBSD) [@DimitryAndric]
@@ -52,6 +51,7 @@ username for an individual isn't available, the brackets will be empty.
 * Peter Smith (ARM) [@smithp35]
 * Pietro Albini (Ferrous Systems; Rust) [@pietroalbini]
 * Serge Guelton (Mozilla) [@serge-sans-paille]
+* Sergey Zverev (Intel) [@offsake]
 * Shayne Hiet-Block (Microsoft) [@GreatKeeper]
 * Tim Penge (Sony) [@tpenge]
 * Tulio Magno Quites Machado Filho (Red Hat) [@tuliom]
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index a40da828ae2a2..d9fa6961032b4 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -38,6 +38,8 @@
 except ImportError:
     if not tags.has("builder-man"):
         raise
+else:
+    myst_enable_extensions = ["substitution"]
 
 # Automatic anchors for markdown titles
 myst_heading_anchors = 6
diff --git a/llvm/include/llvm/ADT/SmallSet.h b/llvm/include/llvm/ADT/SmallSet.h
index 56259ea7cf9d0..97ea4c642bf55 100644
--- a/llvm/include/llvm/ADT/SmallSet.h
+++ b/llvm/include/llvm/ADT/SmallSet.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/iterator.h"
 #include <cstddef>
 #include <functional>
+#include <initializer_list>
 #include <set>
 #include <utility>
 
@@ -147,6 +148,22 @@ class SmallSet {
   using const_iterator = SmallSetIterator<T, N, C>;
 
   SmallSet() = default;
+  SmallSet(const SmallSet &) = default;
+  SmallSet(SmallSet &&) = default;
+
+  template <typename IterT> SmallSet(IterT Begin, IterT End) {
+    insert(Begin, End);
+  }
+
+  template <typename RangeT>
+  explicit SmallSet(const iterator_range<RangeT> &R) {
+    insert(R.begin(), R.end());
+  }
+
+  SmallSet(std::initializer_list<T> L) { insert(L.begin(), L.end()); }
+
+  SmallSet &operator=(const SmallSet &) = default;
+  SmallSet &operator=(SmallSet &&) = default;
 
   [[nodiscard]] bool empty() const { return Vector.empty() && Set.empty(); }
 
diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
index 0a5beb92fcbcc..be040d5eca5f3 100644
--- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h
+++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H
 #define LLVM_ANALYSIS_CTXPROFANALYSIS_H
 
+#include "llvm/ADT/SetVector.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -34,7 +35,7 @@ class PGOContextualProfile {
     uint32_t NextCounterIndex = 0;
     uint32_t NextCallsiteIndex = 0;
     const std::string Name;
-
+    PGOCtxProfContext Index;
     FunctionInfo(StringRef Name) : Name(Name) {}
   };
   std::optional<PGOCtxProfContext::CallTargetMapTy> Profiles;
@@ -49,6 +50,8 @@ class PGOContextualProfile {
   // its state piecemeal.
   PGOContextualProfile() = default;
 
+  void initIndex();
+
 public:
   PGOContextualProfile(const PGOContextualProfile &) = delete;
   PGOContextualProfile(PGOContextualProfile &&) = default;
@@ -63,6 +66,13 @@ class PGOContextualProfile {
     return getDefinedFunctionGUID(F) != 0;
   }
 
+  StringRef getFunctionName(GlobalValue::GUID GUID) const {
+    auto It = FuncInfo.find(GUID);
+    if (It == FuncInfo.end())
+      return "";
+    return It->second.Name;
+  }
+
   uint32_t getNumCounters(const Function &F) const {
     assert(isFunctionKnown(F));
     return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex;
@@ -86,7 +96,7 @@ class PGOContextualProfile {
   using ConstVisitor = function_ref<void(const PGOCtxProfContext &)>;
   using Visitor = function_ref<void(PGOCtxProfContext &)>;
 
-  void update(Visitor, const Function *F = nullptr);
+  void update(Visitor, const Function &F);
   void visit(ConstVisitor, const Function *F = nullptr) const;
 
   const CtxProfFlatProfile flatten() const;
@@ -120,6 +130,11 @@ class CtxProfAnalysis : public AnalysisInfoMixin<CtxProfAnalysis> {
 
   /// Get the step instrumentation associated with a `select`
   static InstrProfIncrementInstStep *getSelectInstrumentation(SelectInst &SI);
+
+  // FIXME: refactor to an advisor model, and separate
+  static void collectIndirectCallPromotionList(
+      CallBase &IC, Result &Profile,
+      SetVector<std::pair<CallBase *, Function *>> &Candidates);
 };
 
 class CtxProfAnalysisPrinterPass
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index decb33e6af6bc..c31e663498d5f 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -492,7 +492,7 @@ class MemoryDependenceResults {
                                    const MemoryLocation &Loc, bool isLoad,
                                    BasicBlock *BB,
                                    SmallVectorImpl<NonLocalDepResult> &Result,
-                                   DenseMap<BasicBlock *, Value *> &Visited,
+                                   SmallDenseMap<BasicBlock *, Value *, 16> &Visited,
                                    bool SkipFirstBlock = false,
                                    bool IsIncomplete = false);
   MemDepResult getNonLocalInfoForBlock(Instruction *QueryInst,
diff --git a/llvm/include/llvm/Analysis/SparsePropagation.h b/llvm/include/llvm/Analysis/SparsePropagation.h
index d5805a7314757..cc79870229873 100644
--- a/llvm/include/llvm/Analysis/SparsePropagation.h
+++ b/llvm/include/llvm/Analysis/SparsePropagation.h
@@ -87,10 +87,9 @@ template <class LatticeKey, class LatticeVal> class AbstractLatticeFunction {
   /// ComputeInstructionState - Compute the LatticeKeys that change as a result
   /// of executing instruction \p I. Their associated LatticeVals are store in
   /// \p ChangedValues.
-  virtual void
-  ComputeInstructionState(Instruction &I,
-                          DenseMap<LatticeKey, LatticeVal> &ChangedValues,
-                          SparseSolver<LatticeKey, LatticeVal> &SS) = 0;
+  virtual void ComputeInstructionState(
+      Instruction &I, SmallDenseMap<LatticeKey, LatticeVal, 16> &ChangedValues,
+      SparseSolver<LatticeKey, LatticeVal> &SS) = 0;
 
   /// PrintLatticeVal - Render the given LatticeVal to the specified stream.
   virtual void PrintLatticeVal(LatticeVal LV, raw_ostream &OS);
@@ -401,7 +400,7 @@ void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::visitPHINode(PHINode &PN) {
   // computed from its incoming values.  For example, SSI form stores its sigma
   // functions as PHINodes with a single incoming value.
   if (LatticeFunc->IsSpecialCasedPHI(&PN)) {
-    DenseMap<LatticeKey, LatticeVal> ChangedValues;
+    SmallDenseMap<LatticeKey, LatticeVal, 16> ChangedValues;
     LatticeFunc->ComputeInstructionState(PN, ChangedValues, *this);
     for (auto &ChangedValue : ChangedValues)
       if (ChangedValue.second != LatticeFunc->getUntrackedVal())
@@ -456,7 +455,7 @@ void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::visitInst(Instruction &I) {
 
   // Otherwise, ask the transfer function what the result is.  If this is
   // something that we care about, remember it.
-  DenseMap<LatticeKey, LatticeVal> ChangedValues;
+  SmallDenseMap<LatticeKey, LatticeVal, 16> ChangedValues;
   LatticeFunc->ComputeInstructionState(I, ChangedValues, *this);
   for (auto &ChangedValue : ChangedValues)
     if (ChangedValue.second != LatticeFunc->getUntrackedVal())
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index ed074ecaebcf5..cb62c86b502c1 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2538,7 +2538,19 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
 
   unsigned getNumberOfParts(Type *Tp) {
     std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
-    return LT.first.isValid() ? *LT.first.getValue() : 0;
+    if (!LT.first.isValid())
+      return 0;
+    // Try to find actual number of parts for non-power-of-2 elements as
+    // ceil(num-of-elements/num-of-subtype-elements).
+    if (auto *FTp = dyn_cast<FixedVectorType>(Tp);
+        Tp && LT.second.isFixedLengthVector() &&
+        !has_single_bit(FTp->getNumElements())) {
+      if (auto *SubTp = dyn_cast_if_present<FixedVectorType>(
+              EVT(LT.second).getTypeForEVT(Tp->getContext()));
+          SubTp && SubTp->getElementType() == FTp->getElementType())
+        return divideCeil(FTp->getNumElements(), SubTp->getNumElements());
+    }
+    return *LT.first.getValue();
   }
 
   InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *,
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 5c1da4fa762e8..997960fcd5d09 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -699,11 +699,6 @@ class LLVM_ABI MachineFunction {
             BBSectionsType == BasicBlockSection::Preset);
   }
 
-  /// Returns true if basic block labels are to be generated for this function.
-  bool hasBBLabels() const {
-    return BBSectionsType == BasicBlockSection::Labels;
-  }
-
   void setBBSectionsType(BasicBlockSection V) { BBSectionsType = V; }
 
   /// Assign IsBeginSection IsEndSection fields for basic blocks in this
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index 1a2f31e199336..9ea0fba1144b1 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -489,6 +489,16 @@ class TargetRegisterInfo : public MCRegisterInfo {
   virtual const MCPhysReg*
   getCalleeSavedRegs(const MachineFunction *MF) const = 0;
 
+  /// Return a null-terminated list of all of the callee-saved registers on
+  /// this target when IPRA is on. The list should include any non-allocatable
+  /// registers that the backend uses and assumes will be saved by all calling
+  /// conventions. This is typically the ISA-standard frame pointer, but could
+  /// include the thread pointer, TOC pointer, or base pointer for different
+  /// targets.
+  virtual const MCPhysReg *getIPRACSRegs(const MachineFunction *MF) const {
+    return nullptr;
+  }
+
   /// Return a mask of call-preserved registers for the given calling convention
   /// on the current function. The mask should include all call-preserved
   /// aliases. This is used by the register allocator to determine which
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index b2a2e11240186..2d8ce66f53ba8 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -563,6 +563,13 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
   def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
   def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
   def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
+  
+  // FP8 fscale
+  def int_aarch64_neon_fp8_fscale : DefaultAttrsIntrinsic<
+                                    [llvm_anyvector_ty],
+                                    [LLVMMatchType<0>,
+                                    LLVMVectorOfBitcastsToInt<0>],
+                                    [IntrNoMem]>;
 }
 
 let TargetPrefix = "aarch64" in {
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 4cd32a0502c66..a03a92b5a97f7 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -862,8 +862,8 @@ class AMDGPUImageDimIntrinsic<AMDGPUDimProfile P_,
       !foreach(arg, P_.DataArgs, arg.Type),    // vdata(VGPR) -- for store/atomic
       !if(P_.IsAtomic, [], [llvm_i32_ty]),     // dmask(imm)
       P_.AddrTypes,                            // vaddr(VGPR)
-      [llvm_v8i32_ty],                         // rsrc(SGPR)
-      !if(P_.IsSample, [llvm_v4i32_ty,         // samp(SGPR)
+      [llvm_any_ty],                           // rsrc(SGPR); Valid types: v4i32 and v8i32
+      !if(P_.IsSample, [llvm_any_ty,           // samp(SGPR);
                         llvm_i1_ty], []),      // unorm(imm)
       [llvm_i32_ty,                            // texfailctrl(imm; bit 0 = tfe, bit 1 = lwe)
        llvm_i32_ty]),                          // auxiliary/cachepolicy(imm):
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 3ce7b8b987ef8..555877e7aaf0e 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -17,9 +17,6 @@ def int_dx_group_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWi
 def int_dx_thread_id_in_group : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>;
 def int_dx_flattened_thread_id_in_group : Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrWillReturn]>;
 
-def int_dx_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">,
-    Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>;
-
 // Create resource handle given binding information. Returns a `target("dx.")`
 // type appropriate for the kind of resource given a register space ID, lower
 // bound and range size of the binding, as well as an index and an indicator
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 737dd6092e218..aa5294f5f9c90 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -4479,22 +4479,6 @@ def int_nvvm_sust_p_3d_v4i32_trap
               "llvm.nvvm.sust.p.3d.v4i32.trap">,
     ClangBuiltin<"__nvvm_sust_p_3d_v4i32_trap">;
 
-
-def int_nvvm_rotate_b32
-  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
-              [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.b32">,
-              ClangBuiltin<"__nvvm_rotate_b32">;
-
-def int_nvvm_rotate_b64
-  : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty],
-             [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.b64">,
-             ClangBuiltin<"__nvvm_rotate_b64">;
-
-def int_nvvm_rotate_right_b64
-  : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty],
-              [IntrNoMem, IntrSpeculatable], "llvm.nvvm.rotate.right.b64">,
-              ClangBuiltin<"__nvvm_rotate_right_b64">;
-
 def int_nvvm_swap_lo_hi_b64
   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty],
               [IntrNoMem, IntrSpeculatable], "llvm.nvvm.swap.lo.hi.b64">,
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index 7ac479f31386f..7ff3d58690ba7 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -15,6 +15,7 @@ let TargetPrefix = "spv" in {
   def int_spv_assign_ptr_type : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>;
   def int_spv_assign_name : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>;
   def int_spv_assign_decoration : Intrinsic<[], [llvm_any_ty, llvm_metadata_ty]>;
+  def int_spv_value_md : Intrinsic<[], [llvm_metadata_ty]>;
 
   def int_spv_track_constant : Intrinsic<[llvm_any_ty], [llvm_any_ty, llvm_metadata_ty]>;
   def int_spv_init_global : Intrinsic<[], [llvm_any_ty, llvm_any_ty]>;
@@ -58,8 +59,6 @@ let TargetPrefix = "spv" in {
 
   // The following intrinsic(s) are mirrored from IntrinsicsDirectX.td for HLSL support.
   def int_spv_thread_id : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrWillReturn]>;
-  def int_spv_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">,
-      Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>;
   def int_spv_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
   def int_spv_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
   def int_spv_frac : DefaultAttrsIntrinsic<[LLVMMatchType<0>], [llvm_anyfloat_ty]>;
diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h
index a617ddecd38a2..6db42878d2e52 100644
--- a/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -268,6 +268,8 @@ class MCRegisterInfo {
   friend class MCRegUnitRootIterator;
   friend class MCRegAliasIterator;
 
+  virtual ~MCRegisterInfo() {}
+
   /// Initialize MCRegisterInfo, called by TableGen
   /// auto-generated routines. *DO NOT USE*.
   void InitMCRegisterInfo(const MCRegisterDesc *D, unsigned NR, unsigned RA,
@@ -418,15 +420,15 @@ class MCRegisterInfo {
   /// number.  Returns -1 if there is no equivalent value.  The second
   /// parameter allows targets to use different numberings for EH info and
   /// debugging info.
-  int getDwarfRegNum(MCRegister RegNum, bool isEH) const;
+  virtual int64_t getDwarfRegNum(MCRegister RegNum, bool isEH) const;
 
   /// Map a dwarf register back to a target register. Returns std::nullopt if
   /// there is no mapping.
-  std::optional<MCRegister> getLLVMRegNum(unsigned RegNum, bool isEH) const;
+  std::optional<MCRegister> getLLVMRegNum(uint64_t RegNum, bool isEH) const;
 
   /// Map a target EH register number to an equivalent DWARF register
   /// number.
-  int getDwarfRegNumFromDwarfEHRegNum(unsigned RegNum) const;
+  int64_t getDwarfRegNumFromDwarfEHRegNum(uint64_t RegNum) const;
 
   /// Map a target register to an equivalent SEH register
   /// number.  Returns LLVM register number if there is no equivalent value.
diff --git a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
index a00c21ddc7d7a..d358041e3a001 100644
--- a/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
+++ b/llvm/include/llvm/ProfileData/PGOCtxProfReader.h
@@ -22,6 +22,58 @@
 #include <map>
 
 namespace llvm {
+class PGOContextualProfile;
+class PGOCtxProfContext;
+
+namespace internal {
+// When we traverse the contextual profile, we typically want to visit contexts
+// pertaining to a specific function. To avoid traversing the whole tree, we
+// want to keep a per-function list - which will be in preorder - of that
+// function's contexts. This happens in PGOContextualProfile. For memory use
+// efficiency, we want to make PGOCtxProfContext an intrusive double-linked list
+// node. We need to handle the cases where PGOCtxProfContext nodes are moved and
+// deleted: in both cases, we need to update the index (==list). We can do that
+// directly from the node in the list, without knowing who the "parent" of the
+// list is. That makes the ADT ilist overkill here. Finally, IndexNode is meant
+// to be an implementation detail of PGOCtxProfContext, and the only reason it's
+// factored out is to avoid implementing move semantics for all its members.
+class IndexNode {
+  // This class' members are intentionally private - it's a convenience
+  // implementation detail.
+  friend class ::llvm::PGOCtxProfContext;
+  friend class ::llvm::PGOContextualProfile;
+
+  IndexNode *Previous = nullptr;
+  IndexNode *Next = nullptr;
+
+  ~IndexNode() {
+    if (Next)
+      Next->Previous = Previous;
+    if (Previous)
+      Previous->Next = Next;
+  }
+
+  IndexNode(const IndexNode &Other) = delete;
+
+  IndexNode(IndexNode &&Other) {
+    // Copy the neighbor info
+    Next = Other.Next;
+    Previous = Other.Previous;
+
+    // Update the neighbors to point to this object
+    if (Other.Next)
+      Other.Next->Previous = this;
+    if (Other.Previous)
+      Other.Previous->Next = this;
+
+    // Make sure the dtor is a noop
+    Other.Next = nullptr;
+    Other.Previous = nullptr;
+  }
+  IndexNode() = default;
+};
+} // namespace internal
+
 /// A node (context) in the loaded contextual profile, suitable for mutation
 /// during IPO passes. We generally expect a fraction of counters and
 /// callsites to be populated. We continue to model counters as vectors, but
@@ -29,13 +81,15 @@ namespace llvm {
 /// there is a small number of indirect targets (usually, 1 for direct calls);
 /// but potentially a large number of callsites, and, as inlining progresses,
 /// the callsite count of a caller will grow.
-class PGOCtxProfContext final {
+class PGOCtxProfContext final : public internal::IndexNode {
 public:
   using CallTargetMapTy = std::map<GlobalValue::GUID, PGOCtxProfContext>;
   using CallsiteMapTy = std::map<uint32_t, CallTargetMapTy>;
 
 private:
   friend class PGOCtxProfileReader;
+  friend class PGOContextualProfile;
+
   GlobalValue::GUID GUID = 0;
   SmallVector<uint64_t, 16> Counters;
   CallsiteMapTy Callsites;
@@ -47,11 +101,15 @@ class PGOCtxProfContext final {
   getOrEmplace(uint32_t Index, GlobalValue::GUID G,
                SmallVectorImpl<uint64_t> &&Counters);
 
+  // Create a bogus context object, used for anchoring the index double linked
+  // list - see IndexNode
+  PGOCtxProfContext() = default;
+
 public:
   PGOCtxProfContext(const PGOCtxProfContext &) = delete;
   PGOCtxProfContext &operator=(const PGOCtxProfContext &) = delete;
   PGOCtxProfContext(PGOCtxProfContext &&) = default;
-  PGOCtxProfContext &operator=(PGOCtxProfContext &&) = default;
+  PGOCtxProfContext &operator=(PGOCtxProfContext &&) = delete;
 
   GlobalValue::GUID guid() const { return GUID; }
   const SmallVectorImpl<uint64_t> &counters() const { return Counters; }
diff --git a/llvm/include/llvm/SandboxIR/Argument.h b/llvm/include/llvm/SandboxIR/Argument.h
new file mode 100644
index 0000000000000..aed886e8f22f2
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/Argument.h
@@ -0,0 +1,38 @@
+//===- Argument.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_ARGUMENT_H
+#define LLVM_SANDBOXIR_ARGUMENT_H
+
+#include "llvm/IR/Argument.h"
+#include "llvm/SandboxIR/Value.h"
+
+namespace llvm::sandboxir {
+
+/// Argument of a sandboxir::Function.
+class Argument : public sandboxir::Value {
+  Argument(llvm::Argument *Arg, sandboxir::Context &Ctx)
+      : Value(ClassID::Argument, Arg, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::Argument;
+  }
+#ifndef NDEBUG
+  void verify() const final {
+    assert(isa<llvm::Argument>(Val) && "Expected Argument!");
+  }
+  void printAsOperand(raw_ostream &OS) const;
+  void dumpOS(raw_ostream &OS) const final;
+#endif
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_ARGUMENT_H
diff --git a/llvm/include/llvm/SandboxIR/BasicBlock.h b/llvm/include/llvm/SandboxIR/BasicBlock.h
new file mode 100644
index 0000000000000..93e79e2a421f9
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/BasicBlock.h
@@ -0,0 +1,112 @@
+//===- BasicBlock.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_BASICBLOCK_H
+#define LLVM_SANDBOXIR_BASICBLOCK_H
+
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/SandboxIR/Value.h"
+
+namespace llvm::sandboxir {
+
+class BasicBlock;
+class Function;
+class Instruction;
+
+/// Iterator for `Instruction`s in a `BasicBlock.
+/// \Returns an sandboxir::Instruction & when derereferenced.
+class BBIterator {
+public:
+  using difference_type = std::ptrdiff_t;
+  using value_type = Instruction;
+  using pointer = value_type *;
+  using reference = value_type &;
+  using iterator_category = std::bidirectional_iterator_tag;
+
+private:
+  llvm::BasicBlock *BB;
+  llvm::BasicBlock::iterator It;
+  Context *Ctx;
+  pointer getInstr(llvm::BasicBlock::iterator It) const;
+
+public:
+  BBIterator() : BB(nullptr), Ctx(nullptr) {}
+  BBIterator(llvm::BasicBlock *BB, llvm::BasicBlock::iterator It, Context *Ctx)
+      : BB(BB), It(It), Ctx(Ctx) {}
+  reference operator*() const { return *getInstr(It); }
+  BBIterator &operator++();
+  BBIterator operator++(int) {
+    auto Copy = *this;
+    ++*this;
+    return Copy;
+  }
+  BBIterator &operator--();
+  BBIterator operator--(int) {
+    auto Copy = *this;
+    --*this;
+    return Copy;
+  }
+  bool operator==(const BBIterator &Other) const {
+    assert(Ctx == Other.Ctx && "BBIterators in different context!");
+    return It == Other.It;
+  }
+  bool operator!=(const BBIterator &Other) const { return !(*this == Other); }
+  /// \Returns the SBInstruction that corresponds to this iterator, or null if
+  /// the instruction is not found in the IR-to-SandboxIR tables.
+  pointer get() const { return getInstr(It); }
+  /// \Returns the parent BB.
+  BasicBlock *getNodeParent() const;
+};
+
+/// Contains a list of sandboxir::Instruction's.
+class BasicBlock : public Value {
+  /// Builds a graph that contains all values in \p BB in their original form
+  /// i.e., no vectorization is taking place here.
+  void buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB);
+  friend class Context;     // For `buildBasicBlockFromIR`
+  friend class Instruction; // For LLVM Val.
+
+  BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
+      : Value(ClassID::Block, BB, SBCtx) {
+    buildBasicBlockFromLLVMIR(BB);
+  }
+
+public:
+  ~BasicBlock() = default;
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == Value::ClassID::Block;
+  }
+  Function *getParent() const;
+  using iterator = BBIterator;
+  iterator begin() const;
+  iterator end() const {
+    auto *BB = cast<llvm::BasicBlock>(Val);
+    return iterator(BB, BB->end(), &Ctx);
+  }
+  std::reverse_iterator<iterator> rbegin() const {
+    return std::make_reverse_iterator(end());
+  }
+  std::reverse_iterator<iterator> rend() const {
+    return std::make_reverse_iterator(begin());
+  }
+  Context &getContext() const { return Ctx; }
+  Instruction *getTerminator() const;
+  bool empty() const { return begin() == end(); }
+  Instruction &front() const;
+  Instruction &back() const;
+
+#ifndef NDEBUG
+  void verify() const final;
+  void dumpOS(raw_ostream &OS) const final;
+#endif
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_BASICBLOCK_H
diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h
new file mode 100644
index 0000000000000..e35d23be6619f
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/Constant.h
@@ -0,0 +1,1285 @@
+//===- Constant.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_CONSTANT_H
+#define LLVM_SANDBOXIR_CONSTANT_H
+
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/GlobalIFunc.h"
+#include "llvm/IR/GlobalObject.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/SandboxIR/Argument.h"
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Type.h"
+#include "llvm/SandboxIR/User.h"
+
+namespace llvm::sandboxir {
+
+class BasicBlock;
+class Function;
+
+class Constant : public sandboxir::User {
+protected:
+  Constant(llvm::Constant *C, sandboxir::Context &SBCtx)
+      : sandboxir::User(ClassID::Constant, C, SBCtx) {}
+  Constant(ClassID ID, llvm::Constant *C, sandboxir::Context &SBCtx)
+      : sandboxir::User(ID, C, SBCtx) {}
+  friend class ConstantInt; // For constructor.
+  friend class Function;    // For constructor
+  friend class Context;     // For constructor.
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const override {
+    return getOperandUseDefault(OpIdx, Verify);
+  }
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    switch (From->getSubclassID()) {
+#define DEF_CONST(ID, CLASS) case ClassID::ID:
+#include "llvm/SandboxIR/SandboxIRValues.def"
+      return true;
+    default:
+      return false;
+    }
+  }
+  sandboxir::Context &getParent() const { return getContext(); }
+  unsigned getUseOperandNo(const Use &Use) const override {
+    return getUseOperandNoDefault(Use);
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::Constant>(Val) && "Expected Constant!");
+  }
+  void dumpOS(raw_ostream &OS) const override;
+#endif
+};
+
+// TODO: This should inherit from ConstantData.
+class ConstantInt : public Constant {
+  ConstantInt(llvm::ConstantInt *C, Context &Ctx)
+      : Constant(ClassID::ConstantInt, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+
+public:
+  static ConstantInt *getTrue(Context &Ctx);
+  static ConstantInt *getFalse(Context &Ctx);
+  static ConstantInt *getBool(Context &Ctx, bool V);
+  static Constant *getTrue(Type *Ty);
+  static Constant *getFalse(Type *Ty);
+  static Constant *getBool(Type *Ty, bool V);
+
+  /// If Ty is a vector type, return a Constant with a splat of the given
+  /// value. Otherwise return a ConstantInt for the given value.
+  static ConstantInt *get(Type *Ty, uint64_t V, bool IsSigned = false);
+
+  /// Return a ConstantInt with the specified integer value for the specified
+  /// type. If the type is wider than 64 bits, the value will be zero-extended
+  /// to fit the type, unless IsSigned is true, in which case the value will
+  /// be interpreted as a 64-bit signed integer and sign-extended to fit
+  /// the type.
+  /// Get a ConstantInt for a specific value.
+  static ConstantInt *get(IntegerType *Ty, uint64_t V, bool IsSigned = false);
+
+  /// Return a ConstantInt with the specified value for the specified type. The
+  /// value V will be canonicalized to a an unsigned APInt. Accessing it with
+  /// either getSExtValue() or getZExtValue() will yield a correctly sized and
+  /// signed value for the type Ty.
+  /// Get a ConstantInt for a specific signed value.
+  static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
+  static Constant *getSigned(Type *Ty, int64_t V);
+
+  /// Return a ConstantInt with the specified value and an implied Type. The
+  /// type is the integer type that corresponds to the bit width of the value.
+  static ConstantInt *get(Context &Ctx, const APInt &V);
+
+  /// Return a ConstantInt constructed from the string strStart with the given
+  /// radix.
+  static ConstantInt *get(IntegerType *Ty, StringRef Str, uint8_t Radix);
+
+  /// If Ty is a vector type, return a Constant with a splat of the given
+  /// value. Otherwise return a ConstantInt for the given value.
+  static Constant *get(Type *Ty, const APInt &V);
+
+  /// Return the constant as an APInt value reference. This allows clients to
+  /// obtain a full-precision copy of the value.
+  /// Return the constant's value.
+  inline const APInt &getValue() const {
+    return cast<llvm::ConstantInt>(Val)->getValue();
+  }
+
+  /// getBitWidth - Return the scalar bitwidth of this constant.
+  unsigned getBitWidth() const {
+    return cast<llvm::ConstantInt>(Val)->getBitWidth();
+  }
+  /// Return the constant as a 64-bit unsigned integer value after it
+  /// has been zero extended as appropriate for the type of this constant. Note
+  /// that this method can assert if the value does not fit in 64 bits.
+  /// Return the zero extended value.
+  inline uint64_t getZExtValue() const {
+    return cast<llvm::ConstantInt>(Val)->getZExtValue();
+  }
+
+  /// Return the constant as a 64-bit integer value after it has been sign
+  /// extended as appropriate for the type of this constant. Note that
+  /// this method can assert if the value does not fit in 64 bits.
+  /// Return the sign extended value.
+  inline int64_t getSExtValue() const {
+    return cast<llvm::ConstantInt>(Val)->getSExtValue();
+  }
+
+  /// Return the constant as an llvm::MaybeAlign.
+  /// Note that this method can assert if the value does not fit in 64 bits or
+  /// is not a power of two.
+  inline MaybeAlign getMaybeAlignValue() const {
+    return cast<llvm::ConstantInt>(Val)->getMaybeAlignValue();
+  }
+
+  /// Return the constant as an llvm::Align, interpreting `0` as `Align(1)`.
+  /// Note that this method can assert if the value does not fit in 64 bits or
+  /// is not a power of two.
+  inline Align getAlignValue() const {
+    return cast<llvm::ConstantInt>(Val)->getAlignValue();
+  }
+
+  /// A helper method that can be used to determine if the constant contained
+  /// within is equal to a constant.  This only works for very small values,
+  /// because this is all that can be represented with all types.
+  /// Determine if this constant's value is same as an unsigned char.
+  bool equalsInt(uint64_t V) const {
+    return cast<llvm::ConstantInt>(Val)->equalsInt(V);
+  }
+
+  /// Variant of the getType() method to always return an IntegerType, which
+  /// reduces the amount of casting needed in parts of the compiler.
+  IntegerType *getIntegerType() const;
+
+  /// This static method returns true if the type Ty is big enough to
+  /// represent the value V. This can be used to avoid having the get method
+  /// assert when V is larger than Ty can represent. Note that there are two
+  /// versions of this method, one for unsigned and one for signed integers.
+  /// Although ConstantInt canonicalizes everything to an unsigned integer,
+  /// the signed version avoids callers having to convert a signed quantity
+  /// to the appropriate unsigned type before calling the method.
+  /// @returns true if V is a valid value for type Ty
+  /// Determine if the value is in range for the given type.
+  static bool isValueValidForType(Type *Ty, uint64_t V);
+  static bool isValueValidForType(Type *Ty, int64_t V);
+
+  bool isNegative() const { return cast<llvm::ConstantInt>(Val)->isNegative(); }
+
+  /// This is just a convenience method to make client code smaller for a
+  /// common code. It also correctly performs the comparison without the
+  /// potential for an assertion from getZExtValue().
+  bool isZero() const { return cast<llvm::ConstantInt>(Val)->isZero(); }
+
+  /// This is just a convenience method to make client code smaller for a
+  /// common case. It also correctly performs the comparison without the
+  /// potential for an assertion from getZExtValue().
+  /// Determine if the value is one.
+  bool isOne() const { return cast<llvm::ConstantInt>(Val)->isOne(); }
+
+  /// This function will return true iff every bit in this constant is set
+  /// to true.
+  /// @returns true iff this constant's bits are all set to true.
+  /// Determine if the value is all ones.
+  bool isMinusOne() const { return cast<llvm::ConstantInt>(Val)->isMinusOne(); }
+
+  /// This function will return true iff this constant represents the largest
+  /// value that may be represented by the constant's type.
+  /// @returns true iff this is the largest value that may be represented
+  /// by this type.
+  /// Determine if the value is maximal.
+  bool isMaxValue(bool IsSigned) const {
+    return cast<llvm::ConstantInt>(Val)->isMaxValue(IsSigned);
+  }
+
+  /// This function will return true iff this constant represents the smallest
+  /// value that may be represented by this constant's type.
+  /// @returns true if this is the smallest value that may be represented by
+  /// this type.
+  /// Determine if the value is minimal.
+  bool isMinValue(bool IsSigned) const {
+    return cast<llvm::ConstantInt>(Val)->isMinValue(IsSigned);
+  }
+
+  /// This function will return true iff this constant represents a value with
+  /// active bits bigger than 64 bits or a value greater than the given uint64_t
+  /// value.
+  /// @returns true iff this constant is greater or equal to the given number.
+  /// Determine if the value is greater or equal to the given number.
+  bool uge(uint64_t Num) const {
+    return cast<llvm::ConstantInt>(Val)->uge(Num);
+  }
+
+  /// getLimitedValue - If the value is smaller than the specified limit,
+  /// return it, otherwise return the limit value.  This causes the value
+  /// to saturate to the limit.
+  /// @returns the min of the value of the constant and the specified value
+  /// Get the constant's value with a saturation limit
+  uint64_t getLimitedValue(uint64_t Limit = ~0ULL) const {
+    return cast<llvm::ConstantInt>(Val)->getLimitedValue(Limit);
+  }
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantInt;
+  }
+  unsigned getUseOperandNo(const Use &Use) const override {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantInt>(Val) && "Expected a ConstantInst!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+// TODO: This should inherit from ConstantData.
+class ConstantFP final : public Constant {
+  ConstantFP(llvm::ConstantFP *C, Context &Ctx)
+      : Constant(ClassID::ConstantFP, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// This returns a ConstantFP, or a vector containing a splat of a ConstantFP,
+  /// for the specified value in the specified type. This should only be used
+  /// for simple constant values like 2.0/1.0 etc, that are known-valid both as
+  /// host double and as the target format.
+  static Constant *get(Type *Ty, double V);
+
+  /// If Ty is a vector type, return a Constant with a splat of the given
+  /// value. Otherwise return a ConstantFP for the given value.
+  static Constant *get(Type *Ty, const APFloat &V);
+
+  static Constant *get(Type *Ty, StringRef Str);
+
+  static ConstantFP *get(const APFloat &V, Context &Ctx);
+
+  static Constant *getNaN(Type *Ty, bool Negative = false,
+                          uint64_t Payload = 0);
+  static Constant *getQNaN(Type *Ty, bool Negative = false,
+                           APInt *Payload = nullptr);
+  static Constant *getSNaN(Type *Ty, bool Negative = false,
+                           APInt *Payload = nullptr);
+  static Constant *getZero(Type *Ty, bool Negative = false);
+
+  static Constant *getNegativeZero(Type *Ty);
+  static Constant *getInfinity(Type *Ty, bool Negative = false);
+
+  /// Return true if Ty is big enough to represent V.
+  static bool isValueValidForType(Type *Ty, const APFloat &V);
+
+  inline const APFloat &getValueAPF() const {
+    return cast<llvm::ConstantFP>(Val)->getValueAPF();
+  }
+  inline const APFloat &getValue() const {
+    return cast<llvm::ConstantFP>(Val)->getValue();
+  }
+
+  /// Return true if the value is positive or negative zero.
+  bool isZero() const { return cast<llvm::ConstantFP>(Val)->isZero(); }
+
+  /// Return true if the sign bit is set.
+  bool isNegative() const { return cast<llvm::ConstantFP>(Val)->isNegative(); }
+
+  /// Return true if the value is infinity
+  bool isInfinity() const { return cast<llvm::ConstantFP>(Val)->isInfinity(); }
+
+  /// Return true if the value is a NaN.
+  bool isNaN() const { return cast<llvm::ConstantFP>(Val)->isNaN(); }
+
+  /// We don't rely on operator== working on double values, as it returns true
+  /// for things that are clearly not equal, like -0.0 and 0.0.
+  /// As such, this method can be used to do an exact bit-for-bit comparison of
+  /// two floating point values.  The version with a double operand is retained
+  /// because it's so convenient to write isExactlyValue(2.0), but please use
+  /// it only for simple constants.
+  bool isExactlyValue(const APFloat &V) const {
+    return cast<llvm::ConstantFP>(Val)->isExactlyValue(V);
+  }
+
+  bool isExactlyValue(double V) const {
+    return cast<llvm::ConstantFP>(Val)->isExactlyValue(V);
+  }
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantFP;
+  }
+
+  // TODO: Better name: getOperandNo(const Use&). Should be private.
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("ConstantFP has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantFP>(Val) && "Expected a ConstantFP!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+/// Base class for aggregate constants (with operands).
+class ConstantAggregate : public Constant {
+protected:
+  ConstantAggregate(ClassID ID, llvm::Constant *C, Context &Ctx)
+      : Constant(ID, C, Ctx) {}
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    auto ID = From->getSubclassID();
+    return ID == ClassID::ConstantVector || ID == ClassID::ConstantStruct ||
+           ID == ClassID::ConstantArray;
+  }
+};
+
+class ConstantArray final : public ConstantAggregate {
+  ConstantArray(llvm::ConstantArray *C, Context &Ctx)
+      : ConstantAggregate(ClassID::ConstantArray, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
+  ArrayType *getType() const;
+
+  // TODO: Missing functions: getType(), getTypeForElements(), getAnon(), get().
+
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::ConstantArray;
+  }
+};
+
+class ConstantStruct final : public ConstantAggregate {
+  ConstantStruct(llvm::ConstantStruct *C, Context &Ctx)
+      : ConstantAggregate(ClassID::ConstantStruct, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static Constant *get(StructType *T, ArrayRef<Constant *> V);
+
+  template <typename... Csts>
+  static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
+  get(StructType *T, Csts *...Vs) {
+    return get(T, ArrayRef<Constant *>({Vs...}));
+  }
+  /// Return an anonymous struct that has the specified elements.
+  /// If the struct is possibly empty, then you must specify a context.
+  static Constant *getAnon(ArrayRef<Constant *> V, bool Packed = false) {
+    return get(getTypeForElements(V, Packed), V);
+  }
+  static Constant *getAnon(Context &Ctx, ArrayRef<Constant *> V,
+                           bool Packed = false) {
+    return get(getTypeForElements(Ctx, V, Packed), V);
+  }
+  /// This version of the method allows an empty list.
+  static StructType *getTypeForElements(Context &Ctx, ArrayRef<Constant *> V,
+                                        bool Packed = false);
+  /// Return an anonymous struct type to use for a constant with the specified
+  /// set of elements. The list must not be empty.
+  static StructType *getTypeForElements(ArrayRef<Constant *> V,
+                                        bool Packed = false) {
+    assert(!V.empty() &&
+           "ConstantStruct::getTypeForElements cannot be called on empty list");
+    return getTypeForElements(V[0]->getContext(), V, Packed);
+  }
+
+  /// Specialization - reduce amount of casting.
+  inline StructType *getType() const {
+    return cast<StructType>(Value::getType());
+  }
+
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::ConstantStruct;
+  }
+};
+
+class ConstantVector final : public ConstantAggregate {
+  ConstantVector(llvm::ConstantVector *C, Context &Ctx)
+      : ConstantAggregate(ClassID::ConstantVector, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  // TODO: Missing functions: getSplat(), getType(), getSplatValue(), get().
+
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::ConstantVector;
+  }
+};
+
+// TODO: Inherit from ConstantData.
+class ConstantAggregateZero final : public Constant {
+  ConstantAggregateZero(llvm::ConstantAggregateZero *C, Context &Ctx)
+      : Constant(ClassID::ConstantAggregateZero, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static ConstantAggregateZero *get(Type *Ty);
+  /// If this CAZ has array or vector type, return a zero with the right element
+  /// type.
+  Constant *getSequentialElement() const;
+  /// If this CAZ has struct type, return a zero with the right element type for
+  /// the specified element.
+  Constant *getStructElement(unsigned Elt) const;
+  /// Return a zero of the right value for the specified GEP index if we can,
+  /// otherwise return null (e.g. if C is a ConstantExpr).
+  Constant *getElementValue(Constant *C) const;
+  /// Return a zero of the right value for the specified GEP index.
+  Constant *getElementValue(unsigned Idx) const;
+  /// Return the number of elements in the array, vector, or struct.
+  ElementCount getElementCount() const {
+    return cast<llvm::ConstantAggregateZero>(Val)->getElementCount();
+  }
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantAggregateZero;
+  }
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("ConstantAggregateZero has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantAggregateZero>(Val) && "Expected a CAZ!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+// TODO: Inherit from ConstantData.
+class ConstantPointerNull final : public Constant {
+  ConstantPointerNull(llvm::ConstantPointerNull *C, Context &Ctx)
+      : Constant(ClassID::ConstantPointerNull, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static ConstantPointerNull *get(PointerType *Ty);
+
+  PointerType *getType() const;
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantPointerNull;
+  }
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("ConstantPointerNull has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantPointerNull>(Val) && "Expected a CPNull!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+// TODO: Inherit from ConstantData.
+class UndefValue : public Constant {
+protected:
+  UndefValue(llvm::UndefValue *C, Context &Ctx)
+      : Constant(ClassID::UndefValue, C, Ctx) {}
+  UndefValue(ClassID ID, llvm::Constant *C, Context &Ctx)
+      : Constant(ID, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Static factory methods - Return an 'undef' object of the specified type.
+  static UndefValue *get(Type *T);
+
+  /// If this Undef has array or vector type, return a undef with the right
+  /// element type.
+  UndefValue *getSequentialElement() const;
+
+  /// If this undef has struct type, return a undef with the right element type
+  /// for the specified element.
+  UndefValue *getStructElement(unsigned Elt) const;
+
+  /// Return an undef of the right value for the specified GEP index if we can,
+  /// otherwise return null (e.g. if C is a ConstantExpr).
+  UndefValue *getElementValue(Constant *C) const;
+
+  /// Return an undef of the right value for the specified GEP index.
+  UndefValue *getElementValue(unsigned Idx) const;
+
+  /// Return the number of elements in the array, vector, or struct.
+  unsigned getNumElements() const {
+    return cast<llvm::UndefValue>(Val)->getNumElements();
+  }
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::UndefValue ||
+           From->getSubclassID() == ClassID::PoisonValue;
+  }
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("UndefValue has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::UndefValue>(Val) && "Expected an UndefValue!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class PoisonValue final : public UndefValue {
+  PoisonValue(llvm::PoisonValue *C, Context &Ctx)
+      : UndefValue(ClassID::PoisonValue, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Static factory methods - Return an 'poison' object of the specified type.
+  static PoisonValue *get(Type *T);
+
+  /// If this poison has array or vector type, return a poison with the right
+  /// element type.
+  PoisonValue *getSequentialElement() const;
+
+  /// If this poison has struct type, return a poison with the right element
+  /// type for the specified element.
+  PoisonValue *getStructElement(unsigned Elt) const;
+
+  /// Return an poison of the right value for the specified GEP index if we can,
+  /// otherwise return null (e.g. if C is a ConstantExpr).
+  PoisonValue *getElementValue(Constant *C) const;
+
+  /// Return an poison of the right value for the specified GEP index.
+  PoisonValue *getElementValue(unsigned Idx) const;
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::PoisonValue;
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::PoisonValue>(Val) && "Expected a PoisonValue!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class GlobalValue : public Constant {
+protected:
+  GlobalValue(ClassID ID, llvm::GlobalValue *C, Context &Ctx)
+      : Constant(ID, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  using LinkageTypes = llvm::GlobalValue::LinkageTypes;
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    switch (From->getSubclassID()) {
+    case ClassID::Function:
+    case ClassID::GlobalVariable:
+    case ClassID::GlobalAlias:
+    case ClassID::GlobalIFunc:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  unsigned getAddressSpace() const {
+    return cast<llvm::GlobalValue>(Val)->getAddressSpace();
+  }
+  bool hasGlobalUnnamedAddr() const {
+    return cast<llvm::GlobalValue>(Val)->hasGlobalUnnamedAddr();
+  }
+
+  /// Returns true if this value's address is not significant in this module.
+  /// This attribute is intended to be used only by the code generator and LTO
+  /// to allow the linker to decide whether the global needs to be in the symbol
+  /// table. It should probably not be used in optimizations, as the value may
+  /// have uses outside the module; use hasGlobalUnnamedAddr() instead.
+  bool hasAtLeastLocalUnnamedAddr() const {
+    return cast<llvm::GlobalValue>(Val)->hasAtLeastLocalUnnamedAddr();
+  }
+
+  using UnnamedAddr = llvm::GlobalValue::UnnamedAddr;
+
+  UnnamedAddr getUnnamedAddr() const {
+    return cast<llvm::GlobalValue>(Val)->getUnnamedAddr();
+  }
+  void setUnnamedAddr(UnnamedAddr V);
+
+  static UnnamedAddr getMinUnnamedAddr(UnnamedAddr A, UnnamedAddr B) {
+    return llvm::GlobalValue::getMinUnnamedAddr(A, B);
+  }
+
+  bool hasComdat() const { return cast<llvm::GlobalValue>(Val)->hasComdat(); }
+
+  // TODO: We need a SandboxIR Comdat if we want to implement getComdat().
+  using VisibilityTypes = llvm::GlobalValue::VisibilityTypes;
+  VisibilityTypes getVisibility() const {
+    return cast<llvm::GlobalValue>(Val)->getVisibility();
+  }
+  bool hasDefaultVisibility() const {
+    return cast<llvm::GlobalValue>(Val)->hasDefaultVisibility();
+  }
+  bool hasHiddenVisibility() const {
+    return cast<llvm::GlobalValue>(Val)->hasHiddenVisibility();
+  }
+  bool hasProtectedVisibility() const {
+    return cast<llvm::GlobalValue>(Val)->hasProtectedVisibility();
+  }
+  void setVisibility(VisibilityTypes V);
+
+  // TODO: Add missing functions.
+};
+
+class GlobalObject : public GlobalValue {
+protected:
+  GlobalObject(ClassID ID, llvm::GlobalObject *C, Context &Ctx)
+      : GlobalValue(ID, C, Ctx) {}
+  friend class Context; // For constructor.
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    return getOperandUseDefault(OpIdx, Verify);
+  }
+
+public:
+  unsigned getUseOperandNo(const Use &Use) const final {
+    return getUseOperandNoDefault(Use);
+  }
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    switch (From->getSubclassID()) {
+    case ClassID::Function:
+    case ClassID::GlobalVariable:
+    case ClassID::GlobalIFunc:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  /// FIXME: Remove this function once transition to Align is over.
+  uint64_t getAlignment() const {
+    return cast<llvm::GlobalObject>(Val)->getAlignment();
+  }
+
+  /// Returns the alignment of the given variable or function.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
+  MaybeAlign getAlign() const {
+    return cast<llvm::GlobalObject>(Val)->getAlign();
+  }
+
+  // TODO: Add missing: setAlignment(Align)
+
+  /// Sets the alignment attribute of the GlobalObject.
+  /// This method will be deprecated as the alignment property should always be
+  /// defined.
+  void setAlignment(MaybeAlign Align);
+
+  unsigned getGlobalObjectSubClassData() const {
+    return cast<llvm::GlobalObject>(Val)->getGlobalObjectSubClassData();
+  }
+
+  void setGlobalObjectSubClassData(unsigned V);
+
+  /// Check if this global has a custom object file section.
+  ///
+  /// This is more efficient than calling getSection() and checking for an empty
+  /// string.
+  bool hasSection() const {
+    return cast<llvm::GlobalObject>(Val)->hasSection();
+  }
+
+  /// Get the custom section of this global if it has one.
+  ///
+  /// If this global does not have a custom section, this will be empty and the
+  /// default object file section (.text, .data, etc) will be used.
+  StringRef getSection() const {
+    return cast<llvm::GlobalObject>(Val)->getSection();
+  }
+
+  /// Change the section for this global.
+  ///
+  /// Setting the section to the empty string tells LLVM to choose an
+  /// appropriate default object file section.
+  void setSection(StringRef S);
+
+  bool hasComdat() const { return cast<llvm::GlobalObject>(Val)->hasComdat(); }
+
+  // TODO: implement get/setComdat(), etc. once we have a sandboxir::Comdat.
+
+  // TODO: We currently don't support Metadata in sandboxir so all
+  // Metadata-related functions are missing.
+
+  using VCallVisibility = llvm::GlobalObject::VCallVisibility;
+
+  VCallVisibility getVCallVisibility() const {
+    return cast<llvm::GlobalObject>(Val)->getVCallVisibility();
+  }
+
+  /// Returns true if the alignment of the value can be unilaterally
+  /// increased.
+  ///
+  /// Note that for functions this is the alignment of the code, not the
+  /// alignment of a function pointer.
+  bool canIncreaseAlignment() const {
+    return cast<llvm::GlobalObject>(Val)->canIncreaseAlignment();
+  }
+};
+
+/// Provides API functions, like getIterator() and getReverseIterator() to
+/// GlobalIFunc, Function, GlobalVariable and GlobalAlias. In LLVM IR these are
+/// provided by ilist_node.
+template <typename GlobalT, typename LLVMGlobalT, typename ParentT,
+          typename LLVMParentT>
+class GlobalWithNodeAPI : public ParentT {
+  /// Helper for mapped_iterator.
+  struct LLVMGVToGV {
+    Context &Ctx;
+    LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
+    GlobalT &operator()(LLVMGlobalT &LLVMGV) const;
+  };
+
+public:
+  GlobalWithNodeAPI(Value::ClassID ID, LLVMParentT *C, Context &Ctx)
+      : ParentT(ID, C, Ctx) {}
+
+  Module *getParent() const {
+    llvm::Module *LLVMM = cast<LLVMGlobalT>(this->Val)->getParent();
+    return this->Ctx.getModule(LLVMM);
+  }
+
+  using iterator = mapped_iterator<
+      decltype(static_cast<LLVMGlobalT *>(nullptr)->getIterator()), LLVMGVToGV>;
+  using reverse_iterator = mapped_iterator<
+      decltype(static_cast<LLVMGlobalT *>(nullptr)->getReverseIterator()),
+      LLVMGVToGV>;
+  iterator getIterator() const {
+    auto *LLVMGV = cast<LLVMGlobalT>(this->Val);
+    LLVMGVToGV ToGV(this->Ctx);
+    return map_iterator(LLVMGV->getIterator(), ToGV);
+  }
+  reverse_iterator getReverseIterator() const {
+    auto *LLVMGV = cast<LLVMGlobalT>(this->Val);
+    LLVMGVToGV ToGV(this->Ctx);
+    return map_iterator(LLVMGV->getReverseIterator(), ToGV);
+  }
+};
+
+class GlobalIFunc final
+    : public GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
+                               llvm::GlobalObject> {
+  GlobalIFunc(llvm::GlobalObject *C, Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::GlobalIFunc, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::GlobalIFunc;
+  }
+
+  // TODO: Missing create() because we don't have a sandboxir::Module yet.
+
+  // TODO: Missing functions: copyAttributesFrom(), removeFromParent(),
+  // eraseFromParent()
+
+  void setResolver(Constant *Resolver);
+
+  Constant *getResolver() const;
+
+  // Return the resolver function after peeling off potential ConstantExpr
+  // indirection.
+  Function *getResolverFunction();
+  const Function *getResolverFunction() const {
+    return const_cast<GlobalIFunc *>(this)->getResolverFunction();
+  }
+
+  static bool isValidLinkage(LinkageTypes L) {
+    return llvm::GlobalIFunc::isValidLinkage(L);
+  }
+
+  // TODO: Missing applyAlongResolverPath().
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::GlobalIFunc>(Val) && "Expected a GlobalIFunc!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class GlobalVariable final
+    : public GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
+                               GlobalObject, llvm::GlobalObject> {
+  GlobalVariable(llvm::GlobalObject *C, Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::GlobalVariable, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  /// Helper for mapped_iterator.
+  struct LLVMGVToGV {
+    Context &Ctx;
+    LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
+    GlobalVariable &operator()(llvm::GlobalVariable &LLVMGV) const;
+  };
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::GlobalVariable;
+  }
+
+  /// Definitions have initializers, declarations don't.
+  ///
+  inline bool hasInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasInitializer();
+  }
+
+  /// hasDefinitiveInitializer - Whether the global variable has an initializer,
+  /// and any other instances of the global (this can happen due to weak
+  /// linkage) are guaranteed to have the same initializer.
+  ///
+  /// Note that if you want to transform a global, you must use
+  /// hasUniqueInitializer() instead, because of the *_odr linkage type.
+  ///
+  /// Example:
+  ///
+  /// @a = global SomeType* null - Initializer is both definitive and unique.
+  ///
+  /// @b = global weak SomeType* null - Initializer is neither definitive nor
+  /// unique.
+  ///
+  /// @c = global weak_odr SomeType* null - Initializer is definitive, but not
+  /// unique.
+  inline bool hasDefinitiveInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasDefinitiveInitializer();
+  }
+
+  /// hasUniqueInitializer - Whether the global variable has an initializer, and
+  /// any changes made to the initializer will turn up in the final executable.
+  inline bool hasUniqueInitializer() const {
+    return cast<llvm::GlobalVariable>(Val)->hasUniqueInitializer();
+  }
+
+  /// getInitializer - Return the initializer for this global variable.  It is
+  /// illegal to call this method if the global is external, because we cannot
+  /// tell what the value is initialized to!
+  ///
+  Constant *getInitializer() const;
+  /// setInitializer - Sets the initializer for this global variable, removing
+  /// any existing initializer if InitVal==NULL. The initializer must have the
+  /// type getValueType().
+  void setInitializer(Constant *InitVal);
+
+  // TODO: Add missing replaceInitializer(). Requires special tracker
+
+  /// If the value is a global constant, its value is immutable throughout the
+  /// runtime execution of the program.  Assigning a value into the constant
+  /// leads to undefined behavior.
+  ///
+  bool isConstant() const {
+    return cast<llvm::GlobalVariable>(Val)->isConstant();
+  }
+  void setConstant(bool V);
+
+  bool isExternallyInitialized() const {
+    return cast<llvm::GlobalVariable>(Val)->isExternallyInitialized();
+  }
+  void setExternallyInitialized(bool Val);
+
+  // TODO: Missing copyAttributesFrom()
+
+  // TODO: Missing removeFromParent(), eraseFromParent(), dropAllReferences()
+
+  // TODO: Missing addDebugInfo(), getDebugInfo()
+
+  // TODO: Missing attribute setter functions: addAttribute(), setAttributes().
+  //       There seems to be no removeAttribute() so we can't undo them.
+
+  /// Return true if the attribute exists.
+  bool hasAttribute(Attribute::AttrKind Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
+  }
+
+  /// Return true if the attribute exists.
+  bool hasAttribute(StringRef Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
+  }
+
+  /// Return true if any attributes exist.
+  bool hasAttributes() const {
+    return cast<llvm::GlobalVariable>(Val)->hasAttributes();
+  }
+
+  /// Return the attribute object.
+  Attribute getAttribute(Attribute::AttrKind Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
+  }
+
+  /// Return the attribute object.
+  Attribute getAttribute(StringRef Kind) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
+  }
+
+  /// Return the attribute set for this global
+  AttributeSet getAttributes() const {
+    return cast<llvm::GlobalVariable>(Val)->getAttributes();
+  }
+
+  /// Return attribute set as list with index.
+  /// FIXME: This may not be required once ValueEnumerators
+  /// in bitcode-writer can enumerate attribute-set.
+  AttributeList getAttributesAsList(unsigned Index) const {
+    return cast<llvm::GlobalVariable>(Val)->getAttributesAsList(Index);
+  }
+
+  /// Check if section name is present
+  bool hasImplicitSection() const {
+    return cast<llvm::GlobalVariable>(Val)->hasImplicitSection();
+  }
+
+  /// Get the custom code model raw value of this global.
+  ///
+  unsigned getCodeModelRaw() const {
+    return cast<llvm::GlobalVariable>(Val)->getCodeModelRaw();
+  }
+
+  /// Get the custom code model of this global if it has one.
+  ///
+  /// If this global does not have a custom code model, the empty instance
+  /// will be returned.
+  std::optional<CodeModel::Model> getCodeModel() const {
+    return cast<llvm::GlobalVariable>(Val)->getCodeModel();
+  }
+
+  // TODO: Missing setCodeModel(). Requires custom tracker.
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::GlobalVariable>(Val) && "Expected a GlobalVariable!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class GlobalAlias final
+    : public GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                               llvm::GlobalValue> {
+  GlobalAlias(llvm::GlobalAlias *C, Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::GlobalAlias, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::GlobalAlias;
+  }
+
+  // TODO: Missing create() due to unimplemented sandboxir::Module.
+
+  // TODO: Missing copyAttributresFrom().
+  // TODO: Missing removeFromParent(), eraseFromParent().
+
+  void setAliasee(Constant *Aliasee);
+  Constant *getAliasee() const;
+
+  const GlobalObject *getAliaseeObject() const;
+  GlobalObject *getAliaseeObject() {
+    return const_cast<GlobalObject *>(
+        static_cast<const GlobalAlias *>(this)->getAliaseeObject());
+  }
+
+  static bool isValidLinkage(LinkageTypes L) {
+    return llvm::GlobalAlias::isValidLinkage(L);
+  }
+};
+
+class NoCFIValue final : public Constant {
+  NoCFIValue(llvm::NoCFIValue *C, Context &Ctx)
+      : Constant(ClassID::NoCFIValue, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    return getOperandUseDefault(OpIdx, Verify);
+  }
+
+public:
+  /// Return a NoCFIValue for the specified function.
+  static NoCFIValue *get(GlobalValue *GV);
+
+  GlobalValue *getGlobalValue() const;
+
+  /// NoCFIValue is always a pointer.
+  PointerType *getType() const;
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::NoCFIValue;
+  }
+
+  unsigned getUseOperandNo(const Use &Use) const final {
+    return getUseOperandNoDefault(Use);
+  }
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::NoCFIValue>(Val) && "Expected a NoCFIValue!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class ConstantPtrAuth final : public Constant {
+  ConstantPtrAuth(llvm::ConstantPtrAuth *C, Context &Ctx)
+      : Constant(ClassID::ConstantPtrAuth, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Return a pointer signed with the specified parameters.
+  static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
+                              ConstantInt *Disc, Constant *AddrDisc);
+  /// The pointer that is signed in this ptrauth signed pointer.
+  Constant *getPointer() const;
+
+  /// The Key ID, an i32 constant.
+  ConstantInt *getKey() const;
+
+  /// The integer discriminator, an i64 constant, or 0.
+  ConstantInt *getDiscriminator() const;
+
+  /// The address discriminator if any, or the null constant.
+  /// If present, this must be a value equivalent to the storage location of
+  /// the only global-initializer user of the ptrauth signed pointer.
+  Constant *getAddrDiscriminator() const;
+
+  /// Whether there is any non-null address discriminator.
+  bool hasAddressDiscriminator() const {
+    return cast<llvm::ConstantPtrAuth>(Val)->hasAddressDiscriminator();
+  }
+
+  /// Whether the address uses a special address discriminator.
+  /// These discriminators can't be used in real pointer-auth values; they
+  /// can only be used in "prototype" values that indicate how some real
+  /// schema is supposed to be produced.
+  bool hasSpecialAddressDiscriminator(uint64_t Value) const {
+    return cast<llvm::ConstantPtrAuth>(Val)->hasSpecialAddressDiscriminator(
+        Value);
+  }
+
+  /// Check whether an authentication operation with key \p Key and (possibly
+  /// blended) discriminator \p Discriminator is known to be compatible with
+  /// this ptrauth signed pointer.
+  bool isKnownCompatibleWith(const Value *Key, const Value *Discriminator,
+                             const DataLayout &DL) const {
+    return cast<llvm::ConstantPtrAuth>(Val)->isKnownCompatibleWith(
+        Key->Val, Discriminator->Val, DL);
+  }
+
+  /// Produce a new ptrauth expression signing the given value using
+  /// the same schema as is stored in one.
+  ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantPtrAuth;
+  }
+};
+
+class ConstantExpr : public Constant {
+  ConstantExpr(llvm::ConstantExpr *C, Context &Ctx)
+      : Constant(ClassID::ConstantExpr, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantExpr;
+  }
+  // TODO: Missing functions.
+};
+
+class BlockAddress final : public Constant {
+  BlockAddress(llvm::BlockAddress *C, Context &Ctx)
+      : Constant(ClassID::BlockAddress, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Return a BlockAddress for the specified function and basic block.
+  static BlockAddress *get(Function *F, BasicBlock *BB);
+
+  /// Return a BlockAddress for the specified basic block.  The basic
+  /// block must be embedded into a function.
+  static BlockAddress *get(BasicBlock *BB);
+
+  /// Lookup an existing \c BlockAddress constant for the given BasicBlock.
+  ///
+  /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress.
+  static BlockAddress *lookup(const BasicBlock *BB);
+
+  Function *getFunction() const;
+  BasicBlock *getBasicBlock() const;
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::BlockAddress;
+  }
+};
+
+class DSOLocalEquivalent final : public Constant {
+  DSOLocalEquivalent(llvm::DSOLocalEquivalent *C, Context &Ctx)
+      : Constant(ClassID::DSOLocalEquivalent, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Return a DSOLocalEquivalent for the specified global value.
+  static DSOLocalEquivalent *get(GlobalValue *GV);
+
+  GlobalValue *getGlobalValue() const;
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::DSOLocalEquivalent;
+  }
+
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("DSOLocalEquivalent has no operands!");
+  }
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::DSOLocalEquivalent>(Val) &&
+           "Expected a DSOLocalEquivalent!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+// TODO: This should inherit from ConstantData.
+class ConstantTokenNone final : public Constant {
+  ConstantTokenNone(llvm::ConstantTokenNone *C, Context &Ctx)
+      : Constant(ClassID::ConstantTokenNone, C, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// Return the ConstantTokenNone.
+  static ConstantTokenNone *get(Context &Ctx);
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantTokenNone;
+  }
+
+  unsigned getUseOperandNo(const Use &Use) const final {
+    llvm_unreachable("ConstantTokenNone has no operands!");
+  }
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantTokenNone>(Val) &&
+           "Expected a ConstantTokenNone!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
+class Function : public GlobalWithNodeAPI<Function, llvm::Function,
+                                          GlobalObject, llvm::GlobalObject> {
+  /// Helper for mapped_iterator.
+  struct LLVMBBToBB {
+    Context &Ctx;
+    LLVMBBToBB(Context &Ctx) : Ctx(Ctx) {}
+    BasicBlock &operator()(llvm::BasicBlock &LLVMBB) const {
+      return *cast<BasicBlock>(Ctx.getValue(&LLVMBB));
+    }
+  };
+  /// Use Context::createFunction() instead.
+  Function(llvm::Function *F, sandboxir::Context &Ctx)
+      : GlobalWithNodeAPI(ClassID::Function, F, Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::Function;
+  }
+
+  Module *getParent() {
+    return Ctx.getModule(cast<llvm::Function>(Val)->getParent());
+  }
+
+  Argument *getArg(unsigned Idx) const {
+    llvm::Argument *Arg = cast<llvm::Function>(Val)->getArg(Idx);
+    return cast<Argument>(Ctx.getValue(Arg));
+  }
+
+  size_t arg_size() const { return cast<llvm::Function>(Val)->arg_size(); }
+  bool arg_empty() const { return cast<llvm::Function>(Val)->arg_empty(); }
+
+  using iterator = mapped_iterator<llvm::Function::iterator, LLVMBBToBB>;
+  iterator begin() const {
+    LLVMBBToBB BBGetter(Ctx);
+    return iterator(cast<llvm::Function>(Val)->begin(), BBGetter);
+  }
+  iterator end() const {
+    LLVMBBToBB BBGetter(Ctx);
+    return iterator(cast<llvm::Function>(Val)->end(), BBGetter);
+  }
+  FunctionType *getFunctionType() const;
+
+#ifndef NDEBUG
+  void verify() const final {
+    assert(isa<llvm::Function>(Val) && "Expected Function!");
+  }
+  void dumpNameAndArgs(raw_ostream &OS) const;
+  void dumpOS(raw_ostream &OS) const final;
+#endif
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_CONSTANT_H
diff --git a/llvm/include/llvm/SandboxIR/Context.h b/llvm/include/llvm/SandboxIR/Context.h
new file mode 100644
index 0000000000000..37f52d74209cb
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/Context.h
@@ -0,0 +1,207 @@
+//===- Context.h ------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_CONTEXT_H
+#define LLVM_SANDBOXIR_CONTEXT_H
+
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/SandboxIR/Tracker.h"
+#include "llvm/SandboxIR/Type.h"
+
+namespace llvm::sandboxir {
+
+class Module;
+class Value;
+class Argument;
+class Constant;
+
+class Context {
+protected:
+  LLVMContext &LLVMCtx;
+  friend class Type;        // For LLVMCtx.
+  friend class PointerType; // For LLVMCtx.
+  friend class CmpInst; // For LLVMCtx. TODO: cleanup when sandboxir::VectorType
+                        // is complete
+  friend class IntegerType;           // For LLVMCtx.
+  friend class StructType;            // For LLVMCtx.
+  friend class ::llvm::TargetExtType; // For LLVMCtx.
+  friend class Region;                // For LLVMCtx.
+
+  Tracker IRTracker;
+
+  /// Maps LLVM Value to the corresponding sandboxir::Value. Owns all
+  /// SandboxIR objects.
+  DenseMap<llvm::Value *, std::unique_ptr<Value>> LLVMValueToValueMap;
+
+  /// Maps an LLVM Module to the corresponding sandboxir::Module.
+  DenseMap<llvm::Module *, std::unique_ptr<Module>> LLVMModuleToModuleMap;
+
+  /// Type has a protected destructor to prohibit the user from managing the
+  /// lifetime of the Type objects. Context is friend of Type, and this custom
+  /// deleter can destroy Type.
+  struct TypeDeleter {
+    void operator()(Type *Ty) { delete Ty; }
+  };
+  /// Maps LLVM Type to the corresonding sandboxir::Type. Owns all Sandbox IR
+  /// Type objects.
+  DenseMap<llvm::Type *, std::unique_ptr<Type, TypeDeleter>> LLVMTypeToTypeMap;
+
+  /// Remove \p V from the maps and returns the unique_ptr.
+  std::unique_ptr<Value> detachLLVMValue(llvm::Value *V);
+  /// Remove \p SBV from all SandboxIR maps and stop owning it. This effectively
+  /// detaches \p V from the underlying IR.
+  std::unique_ptr<Value> detach(Value *V);
+  friend class Instruction; // For detach().
+  /// Take ownership of VPtr and store it in `LLVMValueToValueMap`.
+  Value *registerValue(std::unique_ptr<Value> &&VPtr);
+  friend class EraseFromParent; // For registerValue().
+  /// This is the actual function that creates sandboxir values for \p V,
+  /// and among others handles all instruction types.
+  Value *getOrCreateValueInternal(llvm::Value *V, llvm::User *U = nullptr);
+  /// Get or create a sandboxir::Argument for an existing LLVM IR \p LLVMArg.
+  Argument *getOrCreateArgument(llvm::Argument *LLVMArg);
+  /// Get or create a sandboxir::Value for an existing LLVM IR \p LLVMV.
+  Value *getOrCreateValue(llvm::Value *LLVMV) {
+    return getOrCreateValueInternal(LLVMV, 0);
+  }
+  /// Get or create a sandboxir::Constant from an existing LLVM IR \p LLVMC.
+  Constant *getOrCreateConstant(llvm::Constant *LLVMC);
+
+  // Friends for getOrCreateConstant().
+#define DEF_CONST(ID, CLASS) friend class CLASS;
+#include "llvm/SandboxIR/SandboxIRValues.def"
+
+  /// Create a sandboxir::BasicBlock for an existing LLVM IR \p BB. This will
+  /// also create all contents of the block.
+  BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
+  friend class BasicBlock; // For getOrCreateValue().
+
+  IRBuilder<ConstantFolder> LLVMIRBuilder;
+  auto &getLLVMIRBuilder() { return LLVMIRBuilder; }
+
+  VAArgInst *createVAArgInst(llvm::VAArgInst *SI);
+  friend VAArgInst; // For createVAArgInst()
+  FreezeInst *createFreezeInst(llvm::FreezeInst *SI);
+  friend FreezeInst; // For createFreezeInst()
+  FenceInst *createFenceInst(llvm::FenceInst *SI);
+  friend FenceInst; // For createFenceInst()
+  SelectInst *createSelectInst(llvm::SelectInst *SI);
+  friend SelectInst; // For createSelectInst()
+  InsertElementInst *createInsertElementInst(llvm::InsertElementInst *IEI);
+  friend InsertElementInst; // For createInsertElementInst()
+  ExtractElementInst *createExtractElementInst(llvm::ExtractElementInst *EEI);
+  friend ExtractElementInst; // For createExtractElementInst()
+  ShuffleVectorInst *createShuffleVectorInst(llvm::ShuffleVectorInst *SVI);
+  friend ShuffleVectorInst; // For createShuffleVectorInst()
+  ExtractValueInst *createExtractValueInst(llvm::ExtractValueInst *IVI);
+  friend ExtractValueInst; // For createExtractValueInst()
+  InsertValueInst *createInsertValueInst(llvm::InsertValueInst *IVI);
+  friend InsertValueInst; // For createInsertValueInst()
+  BranchInst *createBranchInst(llvm::BranchInst *I);
+  friend BranchInst; // For createBranchInst()
+  LoadInst *createLoadInst(llvm::LoadInst *LI);
+  friend LoadInst; // For createLoadInst()
+  StoreInst *createStoreInst(llvm::StoreInst *SI);
+  friend StoreInst; // For createStoreInst()
+  ReturnInst *createReturnInst(llvm::ReturnInst *I);
+  friend ReturnInst; // For createReturnInst()
+  CallInst *createCallInst(llvm::CallInst *I);
+  friend CallInst; // For createCallInst()
+  InvokeInst *createInvokeInst(llvm::InvokeInst *I);
+  friend InvokeInst; // For createInvokeInst()
+  CallBrInst *createCallBrInst(llvm::CallBrInst *I);
+  friend CallBrInst; // For createCallBrInst()
+  LandingPadInst *createLandingPadInst(llvm::LandingPadInst *I);
+  friend LandingPadInst; // For createLandingPadInst()
+  CatchPadInst *createCatchPadInst(llvm::CatchPadInst *I);
+  friend CatchPadInst; // For createCatchPadInst()
+  CleanupPadInst *createCleanupPadInst(llvm::CleanupPadInst *I);
+  friend CleanupPadInst; // For createCleanupPadInst()
+  CatchReturnInst *createCatchReturnInst(llvm::CatchReturnInst *I);
+  friend CatchReturnInst; // For createCatchReturnInst()
+  CleanupReturnInst *createCleanupReturnInst(llvm::CleanupReturnInst *I);
+  friend CleanupReturnInst; // For createCleanupReturnInst()
+  GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
+  friend GetElementPtrInst; // For createGetElementPtrInst()
+  CatchSwitchInst *createCatchSwitchInst(llvm::CatchSwitchInst *I);
+  friend CatchSwitchInst; // For createCatchSwitchInst()
+  ResumeInst *createResumeInst(llvm::ResumeInst *I);
+  friend ResumeInst; // For createResumeInst()
+  SwitchInst *createSwitchInst(llvm::SwitchInst *I);
+  friend SwitchInst; // For createSwitchInst()
+  UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
+  friend UnaryOperator; // For createUnaryOperator()
+  BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
+  friend BinaryOperator; // For createBinaryOperator()
+  AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
+  friend AtomicRMWInst; // For createAtomicRMWInst()
+  AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
+  friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
+  AllocaInst *createAllocaInst(llvm::AllocaInst *I);
+  friend AllocaInst; // For createAllocaInst()
+  CastInst *createCastInst(llvm::CastInst *I);
+  friend CastInst; // For createCastInst()
+  PHINode *createPHINode(llvm::PHINode *I);
+  friend PHINode; // For createPHINode()
+  UnreachableInst *createUnreachableInst(llvm::UnreachableInst *UI);
+  friend UnreachableInst; // For createUnreachableInst()
+  CmpInst *createCmpInst(llvm::CmpInst *I);
+  friend CmpInst; // For createCmpInst()
+  ICmpInst *createICmpInst(llvm::ICmpInst *I);
+  friend ICmpInst; // For createICmpInst()
+  FCmpInst *createFCmpInst(llvm::FCmpInst *I);
+  friend FCmpInst; // For createFCmpInst()
+
+public:
+  Context(LLVMContext &LLVMCtx);
+  ~Context();
+
+  Tracker &getTracker() { return IRTracker; }
+  /// Convenience function for `getTracker().save()`
+  void save() { IRTracker.save(); }
+  /// Convenience function for `getTracker().revert()`
+  void revert() { IRTracker.revert(); }
+  /// Convenience function for `getTracker().accept()`
+  void accept() { IRTracker.accept(); }
+
+  sandboxir::Value *getValue(llvm::Value *V) const;
+  const sandboxir::Value *getValue(const llvm::Value *V) const {
+    return getValue(const_cast<llvm::Value *>(V));
+  }
+
+  Module *getModule(llvm::Module *LLVMM) const;
+
+  Module *getOrCreateModule(llvm::Module *LLVMM);
+
+  Type *getType(llvm::Type *LLVMTy) {
+    if (LLVMTy == nullptr)
+      return nullptr;
+    auto Pair = LLVMTypeToTypeMap.insert({LLVMTy, nullptr});
+    auto It = Pair.first;
+    if (Pair.second)
+      It->second = std::unique_ptr<Type, TypeDeleter>(new Type(LLVMTy, *this));
+    return It->second.get();
+  }
+
+  /// Create a sandboxir::Function for an existing LLVM IR \p F, including all
+  /// blocks and instructions.
+  /// This is the main API function for creating Sandbox IR.
+  /// Note: this will not fully populate its parent module. The only globals
+  /// that will be available are those used within the function.
+  Function *createFunction(llvm::Function *F);
+
+  /// Create a sandboxir::Module corresponding to \p LLVMM.
+  Module *createModule(llvm::Module *LLVMM);
+
+  /// \Returns the number of values registered with Context.
+  size_t getNumValues() const { return LLVMValueToValueMap.size(); }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_CONTEXT_H
diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/Instruction.h
similarity index 58%
rename from llvm/include/llvm/SandboxIR/SandboxIR.h
rename to llvm/include/llvm/SandboxIR/Instruction.h
index d99d564ba24e5..f5f5bb5c4443c 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/Instruction.h
@@ -1,1868 +1,26 @@
-//===- SandboxIR.h ----------------------------------------------*- C++ -*-===//
+//===- Instruction.h --------------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// Sandbox IR is a lightweight overlay transactional IR on top of LLVM IR.
-// Features:
-// - You can save/rollback the state of the IR at any time.
-// - Any changes made to Sandbox IR will automatically update the underlying
-//   LLVM IR so both IRs are always in sync.
-// - Feels like LLVM IR, similar API.
-//
-// SandboxIR forms a class hierarchy that resembles that of LLVM IR
-// but is in the `sandboxir` namespace:
-//
-// namespace sandboxir {
-//
-// Value -+- Argument
-//        |
-//        +- BasicBlock
-//        |
-//        +- User ------+- Constant ------ Function
-//                      |
-//                      +- Instruction -+- BinaryOperator
-//                                      |
-//                                      +- BranchInst
-//                                      |
-//                                      +- CastInst --------+- AddrSpaceCastInst
-//                                      |                   |
-//                                      |                   +- BitCastInst
-//                                      |                   |
-//                                      |                   +- FPExtInst
-//                                      |                   |
-//                                      |                   +- FPToSIInst
-//                                      |                   |
-//                                      |                   +- FPToUIInst
-//                                      |                   |
-//                                      |                   +- FPTruncInst
-//                                      |                   |
-//                                      |                   +- IntToPtrInst
-//                                      |                   |
-//                                      |                   +- PtrToIntInst
-//                                      |                   |
-//                                      |                   +- SExtInst
-//                                      |                   |
-//                                      |                   +- SIToFPInst
-//                                      |                   |
-//                                      |                   +- TruncInst
-//                                      |                   |
-//                                      |                   +- UIToFPInst
-//                                      |                   |
-//                                      |                   +- ZExtInst
-//                                      |
-//                                      +- CallBase --------+- CallBrInst
-//                                      |                   |
-//                                      |                   +- CallInst
-//                                      |                   |
-//                                      |                   +- InvokeInst
-//                                      |
-//                                      +- CmpInst ---------+- ICmpInst
-//                                      |                   |
-//                                      |                   +- FCmpInst
-//                                      |
-//                                      +- ExtractElementInst
-//                                      |
-//                                      +- GetElementPtrInst
-//                                      |
-//                                      +- InsertElementInst
-//                                      |
-//                                      +- OpaqueInst
-//                                      |
-//                                      +- PHINode
-//                                      |
-//                                      +- ReturnInst
-//                                      |
-//                                      +- SelectInst
-//                                      |
-//                                      +- ShuffleVectorInst
-//                                      |
-//                                      +- ExtractValueInst
-//                                      |
-//                                      +- InsertValueInst
-//                                      |
-//                                      +- StoreInst
-//                                      |
-//                                      +- UnaryInstruction -+- LoadInst
-//                                      |                    |
-//                                      |                    +- CastInst
-//                                      |
-//                                      +- UnaryOperator
-//                                      |
-//                                      +- UnreachableInst
-//
-// Use
-//
-// } // namespace sandboxir
-//
-
-#ifndef LLVM_SANDBOXIR_SANDBOXIR_H
-#define LLVM_SANDBOXIR_SANDBOXIR_H
-
-#include "llvm/IR/Function.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/User.h"
-#include "llvm/IR/Value.h"
-#include "llvm/SandboxIR/Tracker.h"
-#include "llvm/SandboxIR/Type.h"
-#include "llvm/SandboxIR/Use.h"
-#include "llvm/Support/raw_ostream.h"
-#include <iterator>
-
-namespace llvm {
-
-namespace sandboxir {
-
-class BasicBlock;
-class ConstantInt;
-class ConstantFP;
-class ConstantAggregateZero;
-class ConstantPointerNull;
-class PoisonValue;
-class BlockAddress;
-class DSOLocalEquivalent;
-class ConstantTokenNone;
-class GlobalValue;
-class GlobalObject;
-class GlobalIFunc;
-class GlobalVariable;
-class GlobalAlias;
-class NoCFIValue;
-class ConstantPtrAuth;
-class ConstantExpr;
-class Context;
-class Function;
-class Instruction;
-class VAArgInst;
-class FreezeInst;
-class FenceInst;
-class SelectInst;
-class ExtractElementInst;
-class InsertElementInst;
-class ShuffleVectorInst;
-class ExtractValueInst;
-class InsertValueInst;
-class BranchInst;
-class UnaryInstruction;
-class LoadInst;
-class ReturnInst;
-class StoreInst;
-class User;
-class UnreachableInst;
-class Value;
-class CallBase;
-class CallInst;
-class InvokeInst;
-class CallBrInst;
-class LandingPadInst;
-class FuncletPadInst;
-class CatchPadInst;
-class CleanupPadInst;
-class CatchReturnInst;
-class CleanupReturnInst;
-class GetElementPtrInst;
-class CastInst;
-class PossiblyNonNegInst;
-class PtrToIntInst;
-class BitCastInst;
-class AllocaInst;
-class ResumeInst;
-class CatchSwitchInst;
-class SwitchInst;
-class UnaryOperator;
-class BinaryOperator;
-class PossiblyDisjointInst;
-class AtomicRMWInst;
-class AtomicCmpXchgInst;
-class CmpInst;
-class ICmpInst;
-class FCmpInst;
-
-/// Iterator for the `Use` edges of a User's operands.
-/// \Returns the operand `Use` when dereferenced.
-class OperandUseIterator {
-  sandboxir::Use Use;
-  /// Don't let the user create a non-empty OperandUseIterator.
-  OperandUseIterator(const class Use &Use) : Use(Use) {}
-  friend class User;                                  // For constructor
-#define DEF_INSTR(ID, OPC, CLASS) friend class CLASS; // For constructor
-#include "llvm/SandboxIR/SandboxIRValues.def"
-
-public:
-  using difference_type = std::ptrdiff_t;
-  using value_type = sandboxir::Use;
-  using pointer = value_type *;
-  using reference = value_type &;
-  using iterator_category = std::input_iterator_tag;
-
-  OperandUseIterator() = default;
-  value_type operator*() const;
-  OperandUseIterator &operator++();
-  OperandUseIterator operator++(int) {
-    auto Copy = *this;
-    this->operator++();
-    return Copy;
-  }
-  bool operator==(const OperandUseIterator &Other) const {
-    return Use == Other.Use;
-  }
-  bool operator!=(const OperandUseIterator &Other) const {
-    return !(*this == Other);
-  }
-  OperandUseIterator operator+(unsigned Num) const;
-  OperandUseIterator operator-(unsigned Num) const;
-  int operator-(const OperandUseIterator &Other) const;
-};
-
-/// Iterator for the `Use` edges of a Value's users.
-/// \Returns a `Use` when dereferenced.
-class UserUseIterator {
-  sandboxir::Use Use;
-  /// Don't let the user create a non-empty UserUseIterator.
-  UserUseIterator(const class Use &Use) : Use(Use) {}
-  friend class Value; // For constructor
-
-public:
-  using difference_type = std::ptrdiff_t;
-  using value_type = sandboxir::Use;
-  using pointer = value_type *;
-  using reference = value_type &;
-  using iterator_category = std::input_iterator_tag;
-
-  UserUseIterator() = default;
-  value_type operator*() const { return Use; }
-  UserUseIterator &operator++();
-  bool operator==(const UserUseIterator &Other) const {
-    return Use == Other.Use;
-  }
-  bool operator!=(const UserUseIterator &Other) const {
-    return !(*this == Other);
-  }
-  const sandboxir::Use &getUse() const { return Use; }
-};
-
-/// A SandboxIR Value has users. This is the base class.
-class Value {
-public:
-  enum class ClassID : unsigned {
-#define DEF_VALUE(ID, CLASS) ID,
-#define DEF_USER(ID, CLASS) ID,
-#define DEF_CONST(ID, CLASS) ID,
-#define DEF_INSTR(ID, OPC, CLASS) ID,
-#include "llvm/SandboxIR/SandboxIRValues.def"
-  };
-
-protected:
-  static const char *getSubclassIDStr(ClassID ID) {
-    switch (ID) {
-#define DEF_VALUE(ID, CLASS)                                                   \
-  case ClassID::ID:                                                            \
-    return #ID;
-#define DEF_USER(ID, CLASS)                                                    \
-  case ClassID::ID:                                                            \
-    return #ID;
-#define DEF_CONST(ID, CLASS)                                                   \
-  case ClassID::ID:                                                            \
-    return #ID;
-#define DEF_INSTR(ID, OPC, CLASS)                                              \
-  case ClassID::ID:                                                            \
-    return #ID;
-#include "llvm/SandboxIR/SandboxIRValues.def"
-    }
-    llvm_unreachable("Unimplemented ID");
-  }
-
-  /// For isa/dyn_cast.
-  ClassID SubclassID;
-#ifndef NDEBUG
-  /// A unique ID used for forming the name (used for debugging).
-  unsigned UID;
-#endif
-  /// The LLVM Value that corresponds to this SandboxIR Value.
-  /// NOTE: Some sandboxir Instructions, like Packs, may include more than one
-  /// value and in these cases `Val` points to the last instruction in program
-  /// order.
-  llvm::Value *Val = nullptr;
-
-  friend class Context;               // For getting `Val`.
-  friend class User;                  // For getting `Val`.
-  friend class Use;                   // For getting `Val`.
-  friend class VAArgInst;             // For getting `Val`.
-  friend class FreezeInst;            // For getting `Val`.
-  friend class FenceInst;             // For getting `Val`.
-  friend class SelectInst;            // For getting `Val`.
-  friend class ExtractElementInst;    // For getting `Val`.
-  friend class InsertElementInst;     // For getting `Val`.
-  friend class ShuffleVectorInst;     // For getting `Val`.
-  friend class ExtractValueInst;      // For getting `Val`.
-  friend class InsertValueInst;       // For getting `Val`.
-  friend class BranchInst;            // For getting `Val`.
-  friend class LoadInst;              // For getting `Val`.
-  friend class StoreInst;             // For getting `Val`.
-  friend class ReturnInst;            // For getting `Val`.
-  friend class CallBase;              // For getting `Val`.
-  friend class CallInst;              // For getting `Val`.
-  friend class InvokeInst;            // For getting `Val`.
-  friend class CallBrInst;            // For getting `Val`.
-  friend class LandingPadInst;        // For getting `Val`.
-  friend class FuncletPadInst;        // For getting `Val`.
-  friend class CatchPadInst;          // For getting `Val`.
-  friend class CleanupPadInst;        // For getting `Val`.
-  friend class CatchReturnInst;       // For getting `Val`.
-  friend class GetElementPtrInst;     // For getting `Val`.
-  friend class ResumeInst;            // For getting `Val`.
-  friend class CatchSwitchInst;       // For getting `Val`.
-  friend class CleanupReturnInst;     // For getting `Val`.
-  friend class SwitchInst;            // For getting `Val`.
-  friend class UnaryOperator;         // For getting `Val`.
-  friend class BinaryOperator;        // For getting `Val`.
-  friend class AtomicRMWInst;         // For getting `Val`.
-  friend class AtomicCmpXchgInst;     // For getting `Val`.
-  friend class AllocaInst;            // For getting `Val`.
-  friend class CastInst;              // For getting `Val`.
-  friend class PHINode;               // For getting `Val`.
-  friend class UnreachableInst;       // For getting `Val`.
-  friend class CatchSwitchAddHandler; // For `Val`.
-  friend class CmpInst;               // For getting `Val`.
-  friend class ConstantArray;         // For `Val`.
-  friend class ConstantStruct;        // For `Val`.
-  friend class ConstantAggregateZero; // For `Val`.
-  friend class ConstantPointerNull;   // For `Val`.
-  friend class UndefValue;            // For `Val`.
-  friend class PoisonValue;           // For `Val`.
-  friend class BlockAddress;          // For `Val`.
-  friend class GlobalValue;           // For `Val`.
-  friend class DSOLocalEquivalent;    // For `Val`.
-  friend class GlobalObject;          // For `Val`.
-  friend class GlobalIFunc;           // For `Val`.
-  friend class GlobalVariable;        // For `Val`.
-  friend class GlobalAlias;           // For `Val`.
-  friend class NoCFIValue;            // For `Val`.
-  friend class ConstantPtrAuth;       // For `Val`.
-  friend class ConstantExpr;          // For `Val`.
-  friend class Utils;                 // For `Val`.
-
-  // Region needs to manipulate metadata in the underlying LLVM Value, we don't
-  // expose metadata in sandboxir.
-  friend class Region;
-
-  /// All values point to the context.
-  Context &Ctx;
-  // This is used by eraseFromParent().
-  void clearValue() { Val = nullptr; }
-  template <typename ItTy, typename SBTy> friend class LLVMOpUserItToSBTy;
-
-  Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx);
-  /// Disable copies.
-  Value(const Value &) = delete;
-  Value &operator=(const Value &) = delete;
-
-public:
-  virtual ~Value() = default;
-  ClassID getSubclassID() const { return SubclassID; }
-
-  using use_iterator = UserUseIterator;
-  using const_use_iterator = UserUseIterator;
-
-  use_iterator use_begin();
-  const_use_iterator use_begin() const {
-    return const_cast<Value *>(this)->use_begin();
-  }
-  use_iterator use_end() { return use_iterator(Use(nullptr, nullptr, Ctx)); }
-  const_use_iterator use_end() const {
-    return const_cast<Value *>(this)->use_end();
-  }
-
-  iterator_range<use_iterator> uses() {
-    return make_range<use_iterator>(use_begin(), use_end());
-  }
-  iterator_range<const_use_iterator> uses() const {
-    return make_range<const_use_iterator>(use_begin(), use_end());
-  }
-
-  /// Helper for mapped_iterator.
-  struct UseToUser {
-    User *operator()(const Use &Use) const { return &*Use.getUser(); }
-  };
-
-  using user_iterator = mapped_iterator<sandboxir::UserUseIterator, UseToUser>;
-  using const_user_iterator = user_iterator;
-
-  user_iterator user_begin();
-  user_iterator user_end() {
-    return user_iterator(Use(nullptr, nullptr, Ctx), UseToUser());
-  }
-  const_user_iterator user_begin() const {
-    return const_cast<Value *>(this)->user_begin();
-  }
-  const_user_iterator user_end() const {
-    return const_cast<Value *>(this)->user_end();
-  }
-
-  iterator_range<user_iterator> users() {
-    return make_range<user_iterator>(user_begin(), user_end());
-  }
-  iterator_range<const_user_iterator> users() const {
-    return make_range<const_user_iterator>(user_begin(), user_end());
-  }
-  /// \Returns the number of user edges (not necessarily to unique users).
-  /// WARNING: This is a linear-time operation.
-  unsigned getNumUses() const;
-  /// Return true if this value has N uses or more.
-  /// This is logically equivalent to getNumUses() >= N.
-  /// WARNING: This can be expensive, as it is linear to the number of users.
-  bool hasNUsesOrMore(unsigned Num) const {
-    unsigned Cnt = 0;
-    for (auto It = use_begin(), ItE = use_end(); It != ItE; ++It) {
-      if (++Cnt >= Num)
-        return true;
-    }
-    return false;
-  }
-  /// Return true if this Value has exactly N uses.
-  bool hasNUses(unsigned Num) const {
-    unsigned Cnt = 0;
-    for (auto It = use_begin(), ItE = use_end(); It != ItE; ++It) {
-      if (++Cnt > Num)
-        return false;
-    }
-    return Cnt == Num;
-  }
-
-  Type *getType() const;
-
-  Context &getContext() const { return Ctx; }
-
-  void replaceUsesWithIf(Value *OtherV,
-                         llvm::function_ref<bool(const Use &)> ShouldReplace);
-  void replaceAllUsesWith(Value *Other);
-
-  /// \Returns the LLVM IR name of the bottom-most LLVM value.
-  StringRef getName() const { return Val->getName(); }
-
-#ifndef NDEBUG
-  /// Should crash if there is something wrong with the instruction.
-  virtual void verify() const = 0;
-  /// Returns the unique id in the form 'SB<number>.' like 'SB1.'
-  std::string getUid() const;
-  virtual void dumpCommonHeader(raw_ostream &OS) const;
-  void dumpCommonFooter(raw_ostream &OS) const;
-  void dumpCommonPrefix(raw_ostream &OS) const;
-  void dumpCommonSuffix(raw_ostream &OS) const;
-  void printAsOperandCommon(raw_ostream &OS) const;
-  friend raw_ostream &operator<<(raw_ostream &OS, const sandboxir::Value &V) {
-    V.dumpOS(OS);
-    return OS;
-  }
-  virtual void dumpOS(raw_ostream &OS) const = 0;
-  LLVM_DUMP_METHOD void dump() const;
-#endif
-};
-
-/// Argument of a sandboxir::Function.
-class Argument : public sandboxir::Value {
-  Argument(llvm::Argument *Arg, sandboxir::Context &Ctx)
-      : sandboxir::Value(ClassID::Argument, Arg, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::Argument;
-  }
-#ifndef NDEBUG
-  void verify() const final {
-    assert(isa<llvm::Argument>(Val) && "Expected Argument!");
-  }
-  void printAsOperand(raw_ostream &OS) const;
-  void dumpOS(raw_ostream &OS) const final;
-#endif
-};
-
-/// A sandboxir::User has operands.
-class User : public Value {
-protected:
-  User(ClassID ID, llvm::Value *V, Context &Ctx) : Value(ID, V, Ctx) {}
-
-  /// \Returns the Use edge that corresponds to \p OpIdx.
-  /// Note: This is the default implementation that works for instructions that
-  /// match the underlying LLVM instruction. All others should use a different
-  /// implementation.
-  Use getOperandUseDefault(unsigned OpIdx, bool Verify) const;
-  /// \Returns the Use for the \p OpIdx'th operand. This is virtual to allow
-  /// instructions to deviate from the LLVM IR operands, which is a requirement
-  /// for sandboxir Instructions that consist of more than one LLVM Instruction.
-  virtual Use getOperandUseInternal(unsigned OpIdx, bool Verify) const = 0;
-  friend class OperandUseIterator; // for getOperandUseInternal()
-
-  /// The default implementation works only for single-LLVMIR-instruction
-  /// Users and only if they match exactly the LLVM instruction.
-  unsigned getUseOperandNoDefault(const Use &Use) const {
-    return Use.LLVMUse->getOperandNo();
-  }
-  /// \Returns the operand index of \p Use.
-  virtual unsigned getUseOperandNo(const Use &Use) const = 0;
-  friend unsigned Use::getOperandNo() const; // For getUseOperandNo()
-
-  void swapOperandsInternal(unsigned OpIdxA, unsigned OpIdxB) {
-    assert(OpIdxA < getNumOperands() && "OpIdxA out of bounds!");
-    assert(OpIdxB < getNumOperands() && "OpIdxB out of bounds!");
-    auto UseA = getOperandUse(OpIdxA);
-    auto UseB = getOperandUse(OpIdxB);
-    UseA.swap(UseB);
-  }
-
-#ifndef NDEBUG
-  void verifyUserOfLLVMUse(const llvm::Use &Use) const;
-#endif // NDEBUG
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const Value *From);
-  using op_iterator = OperandUseIterator;
-  using const_op_iterator = OperandUseIterator;
-  using op_range = iterator_range<op_iterator>;
-  using const_op_range = iterator_range<const_op_iterator>;
-
-  virtual op_iterator op_begin() {
-    assert(isa<llvm::User>(Val) && "Expect User value!");
-    return op_iterator(getOperandUseInternal(0, /*Verify=*/false));
-  }
-  virtual op_iterator op_end() {
-    assert(isa<llvm::User>(Val) && "Expect User value!");
-    return op_iterator(
-        getOperandUseInternal(getNumOperands(), /*Verify=*/false));
-  }
-  virtual const_op_iterator op_begin() const {
-    return const_cast<User *>(this)->op_begin();
-  }
-  virtual const_op_iterator op_end() const {
-    return const_cast<User *>(this)->op_end();
-  }
-
-  op_range operands() { return make_range<op_iterator>(op_begin(), op_end()); }
-  const_op_range operands() const {
-    return make_range<const_op_iterator>(op_begin(), op_end());
-  }
-  Value *getOperand(unsigned OpIdx) const { return getOperandUse(OpIdx).get(); }
-  /// \Returns the operand edge for \p OpIdx. NOTE: This should also work for
-  /// OpIdx == getNumOperands(), which is used for op_end().
-  Use getOperandUse(unsigned OpIdx) const {
-    return getOperandUseInternal(OpIdx, /*Verify=*/true);
-  }
-  virtual unsigned getNumOperands() const {
-    return isa<llvm::User>(Val) ? cast<llvm::User>(Val)->getNumOperands() : 0;
-  }
-
-  virtual void setOperand(unsigned OperandIdx, Value *Operand);
-  /// Replaces any operands that match \p FromV with \p ToV. Returns whether any
-  /// operands were replaced.
-  bool replaceUsesOfWith(Value *FromV, Value *ToV);
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::User>(Val) && "Expected User!");
-  }
-  void dumpCommonHeader(raw_ostream &OS) const final;
-  void dumpOS(raw_ostream &OS) const override {
-    // TODO: Remove this tmp implementation once we get the Instruction classes.
-  }
-#endif
-};
-
-class Constant : public sandboxir::User {
-protected:
-  Constant(llvm::Constant *C, sandboxir::Context &SBCtx)
-      : sandboxir::User(ClassID::Constant, C, SBCtx) {}
-  Constant(ClassID ID, llvm::Constant *C, sandboxir::Context &SBCtx)
-      : sandboxir::User(ID, C, SBCtx) {}
-  friend class ConstantInt; // For constructor.
-  friend class Function;    // For constructor
-  friend class Context;     // For constructor.
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const override {
-    return getOperandUseDefault(OpIdx, Verify);
-  }
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    switch (From->getSubclassID()) {
-#define DEF_CONST(ID, CLASS) case ClassID::ID:
-#include "llvm/SandboxIR/SandboxIRValues.def"
-      return true;
-    default:
-      return false;
-    }
-  }
-  sandboxir::Context &getParent() const { return getContext(); }
-  unsigned getUseOperandNo(const Use &Use) const override {
-    return getUseOperandNoDefault(Use);
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::Constant>(Val) && "Expected Constant!");
-  }
-  void dumpOS(raw_ostream &OS) const override;
-#endif
-};
-
-// TODO: This should inherit from ConstantData.
-class ConstantInt : public Constant {
-  ConstantInt(llvm::ConstantInt *C, Context &Ctx)
-      : Constant(ClassID::ConstantInt, C, Ctx) {}
-  friend class Context; // For constructor.
-
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
-    llvm_unreachable("ConstantInt has no operands!");
-  }
-
-public:
-  static ConstantInt *getTrue(Context &Ctx);
-  static ConstantInt *getFalse(Context &Ctx);
-  static ConstantInt *getBool(Context &Ctx, bool V);
-  static Constant *getTrue(Type *Ty);
-  static Constant *getFalse(Type *Ty);
-  static Constant *getBool(Type *Ty, bool V);
-
-  /// If Ty is a vector type, return a Constant with a splat of the given
-  /// value. Otherwise return a ConstantInt for the given value.
-  static ConstantInt *get(Type *Ty, uint64_t V, bool IsSigned = false);
-
-  /// Return a ConstantInt with the specified integer value for the specified
-  /// type. If the type is wider than 64 bits, the value will be zero-extended
-  /// to fit the type, unless IsSigned is true, in which case the value will
-  /// be interpreted as a 64-bit signed integer and sign-extended to fit
-  /// the type.
-  /// Get a ConstantInt for a specific value.
-  static ConstantInt *get(IntegerType *Ty, uint64_t V, bool IsSigned = false);
-
-  /// Return a ConstantInt with the specified value for the specified type. The
-  /// value V will be canonicalized to a an unsigned APInt. Accessing it with
-  /// either getSExtValue() or getZExtValue() will yield a correctly sized and
-  /// signed value for the type Ty.
-  /// Get a ConstantInt for a specific signed value.
-  static ConstantInt *getSigned(IntegerType *Ty, int64_t V);
-  static Constant *getSigned(Type *Ty, int64_t V);
-
-  /// Return a ConstantInt with the specified value and an implied Type. The
-  /// type is the integer type that corresponds to the bit width of the value.
-  static ConstantInt *get(Context &Ctx, const APInt &V);
-
-  /// Return a ConstantInt constructed from the string strStart with the given
-  /// radix.
-  static ConstantInt *get(IntegerType *Ty, StringRef Str, uint8_t Radix);
-
-  /// If Ty is a vector type, return a Constant with a splat of the given
-  /// value. Otherwise return a ConstantInt for the given value.
-  static Constant *get(Type *Ty, const APInt &V);
-
-  /// Return the constant as an APInt value reference. This allows clients to
-  /// obtain a full-precision copy of the value.
-  /// Return the constant's value.
-  inline const APInt &getValue() const {
-    return cast<llvm::ConstantInt>(Val)->getValue();
-  }
-
-  /// getBitWidth - Return the scalar bitwidth of this constant.
-  unsigned getBitWidth() const {
-    return cast<llvm::ConstantInt>(Val)->getBitWidth();
-  }
-  /// Return the constant as a 64-bit unsigned integer value after it
-  /// has been zero extended as appropriate for the type of this constant. Note
-  /// that this method can assert if the value does not fit in 64 bits.
-  /// Return the zero extended value.
-  inline uint64_t getZExtValue() const {
-    return cast<llvm::ConstantInt>(Val)->getZExtValue();
-  }
-
-  /// Return the constant as a 64-bit integer value after it has been sign
-  /// extended as appropriate for the type of this constant. Note that
-  /// this method can assert if the value does not fit in 64 bits.
-  /// Return the sign extended value.
-  inline int64_t getSExtValue() const {
-    return cast<llvm::ConstantInt>(Val)->getSExtValue();
-  }
-
-  /// Return the constant as an llvm::MaybeAlign.
-  /// Note that this method can assert if the value does not fit in 64 bits or
-  /// is not a power of two.
-  inline MaybeAlign getMaybeAlignValue() const {
-    return cast<llvm::ConstantInt>(Val)->getMaybeAlignValue();
-  }
-
-  /// Return the constant as an llvm::Align, interpreting `0` as `Align(1)`.
-  /// Note that this method can assert if the value does not fit in 64 bits or
-  /// is not a power of two.
-  inline Align getAlignValue() const {
-    return cast<llvm::ConstantInt>(Val)->getAlignValue();
-  }
-
-  /// A helper method that can be used to determine if the constant contained
-  /// within is equal to a constant.  This only works for very small values,
-  /// because this is all that can be represented with all types.
-  /// Determine if this constant's value is same as an unsigned char.
-  bool equalsInt(uint64_t V) const {
-    return cast<llvm::ConstantInt>(Val)->equalsInt(V);
-  }
-
-  /// Variant of the getType() method to always return an IntegerType, which
-  /// reduces the amount of casting needed in parts of the compiler.
-  IntegerType *getIntegerType() const;
-
-  /// This static method returns true if the type Ty is big enough to
-  /// represent the value V. This can be used to avoid having the get method
-  /// assert when V is larger than Ty can represent. Note that there are two
-  /// versions of this method, one for unsigned and one for signed integers.
-  /// Although ConstantInt canonicalizes everything to an unsigned integer,
-  /// the signed version avoids callers having to convert a signed quantity
-  /// to the appropriate unsigned type before calling the method.
-  /// @returns true if V is a valid value for type Ty
-  /// Determine if the value is in range for the given type.
-  static bool isValueValidForType(Type *Ty, uint64_t V);
-  static bool isValueValidForType(Type *Ty, int64_t V);
-
-  bool isNegative() const { return cast<llvm::ConstantInt>(Val)->isNegative(); }
-
-  /// This is just a convenience method to make client code smaller for a
-  /// common code. It also correctly performs the comparison without the
-  /// potential for an assertion from getZExtValue().
-  bool isZero() const { return cast<llvm::ConstantInt>(Val)->isZero(); }
-
-  /// This is just a convenience method to make client code smaller for a
-  /// common case. It also correctly performs the comparison without the
-  /// potential for an assertion from getZExtValue().
-  /// Determine if the value is one.
-  bool isOne() const { return cast<llvm::ConstantInt>(Val)->isOne(); }
-
-  /// This function will return true iff every bit in this constant is set
-  /// to true.
-  /// @returns true iff this constant's bits are all set to true.
-  /// Determine if the value is all ones.
-  bool isMinusOne() const { return cast<llvm::ConstantInt>(Val)->isMinusOne(); }
-
-  /// This function will return true iff this constant represents the largest
-  /// value that may be represented by the constant's type.
-  /// @returns true iff this is the largest value that may be represented
-  /// by this type.
-  /// Determine if the value is maximal.
-  bool isMaxValue(bool IsSigned) const {
-    return cast<llvm::ConstantInt>(Val)->isMaxValue(IsSigned);
-  }
-
-  /// This function will return true iff this constant represents the smallest
-  /// value that may be represented by this constant's type.
-  /// @returns true if this is the smallest value that may be represented by
-  /// this type.
-  /// Determine if the value is minimal.
-  bool isMinValue(bool IsSigned) const {
-    return cast<llvm::ConstantInt>(Val)->isMinValue(IsSigned);
-  }
-
-  /// This function will return true iff this constant represents a value with
-  /// active bits bigger than 64 bits or a value greater than the given uint64_t
-  /// value.
-  /// @returns true iff this constant is greater or equal to the given number.
-  /// Determine if the value is greater or equal to the given number.
-  bool uge(uint64_t Num) const {
-    return cast<llvm::ConstantInt>(Val)->uge(Num);
-  }
-
-  /// getLimitedValue - If the value is smaller than the specified limit,
-  /// return it, otherwise return the limit value.  This causes the value
-  /// to saturate to the limit.
-  /// @returns the min of the value of the constant and the specified value
-  /// Get the constant's value with a saturation limit
-  uint64_t getLimitedValue(uint64_t Limit = ~0ULL) const {
-    return cast<llvm::ConstantInt>(Val)->getLimitedValue(Limit);
-  }
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantInt;
-  }
-  unsigned getUseOperandNo(const Use &Use) const override {
-    llvm_unreachable("ConstantInt has no operands!");
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::ConstantInt>(Val) && "Expected a ConstantInst!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-// TODO: This should inherit from ConstantData.
-class ConstantFP final : public Constant {
-  ConstantFP(llvm::ConstantFP *C, Context &Ctx)
-      : Constant(ClassID::ConstantFP, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// This returns a ConstantFP, or a vector containing a splat of a ConstantFP,
-  /// for the specified value in the specified type. This should only be used
-  /// for simple constant values like 2.0/1.0 etc, that are known-valid both as
-  /// host double and as the target format.
-  static Constant *get(Type *Ty, double V);
-
-  /// If Ty is a vector type, return a Constant with a splat of the given
-  /// value. Otherwise return a ConstantFP for the given value.
-  static Constant *get(Type *Ty, const APFloat &V);
-
-  static Constant *get(Type *Ty, StringRef Str);
-
-  static ConstantFP *get(const APFloat &V, Context &Ctx);
-
-  static Constant *getNaN(Type *Ty, bool Negative = false,
-                          uint64_t Payload = 0);
-  static Constant *getQNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getSNaN(Type *Ty, bool Negative = false,
-                           APInt *Payload = nullptr);
-  static Constant *getZero(Type *Ty, bool Negative = false);
-
-  static Constant *getNegativeZero(Type *Ty);
-  static Constant *getInfinity(Type *Ty, bool Negative = false);
-
-  /// Return true if Ty is big enough to represent V.
-  static bool isValueValidForType(Type *Ty, const APFloat &V);
-
-  inline const APFloat &getValueAPF() const {
-    return cast<llvm::ConstantFP>(Val)->getValueAPF();
-  }
-  inline const APFloat &getValue() const {
-    return cast<llvm::ConstantFP>(Val)->getValue();
-  }
-
-  /// Return true if the value is positive or negative zero.
-  bool isZero() const { return cast<llvm::ConstantFP>(Val)->isZero(); }
-
-  /// Return true if the sign bit is set.
-  bool isNegative() const { return cast<llvm::ConstantFP>(Val)->isNegative(); }
-
-  /// Return true if the value is infinity
-  bool isInfinity() const { return cast<llvm::ConstantFP>(Val)->isInfinity(); }
-
-  /// Return true if the value is a NaN.
-  bool isNaN() const { return cast<llvm::ConstantFP>(Val)->isNaN(); }
-
-  /// We don't rely on operator== working on double values, as it returns true
-  /// for things that are clearly not equal, like -0.0 and 0.0.
-  /// As such, this method can be used to do an exact bit-for-bit comparison of
-  /// two floating point values.  The version with a double operand is retained
-  /// because it's so convenient to write isExactlyValue(2.0), but please use
-  /// it only for simple constants.
-  bool isExactlyValue(const APFloat &V) const {
-    return cast<llvm::ConstantFP>(Val)->isExactlyValue(V);
-  }
-
-  bool isExactlyValue(double V) const {
-    return cast<llvm::ConstantFP>(Val)->isExactlyValue(V);
-  }
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantFP;
-  }
-
-  // TODO: Better name: getOperandNo(const Use&). Should be private.
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("ConstantFP has no operands!");
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::ConstantFP>(Val) && "Expected a ConstantFP!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-/// Base class for aggregate constants (with operands).
-class ConstantAggregate : public Constant {
-protected:
-  ConstantAggregate(ClassID ID, llvm::Constant *C, Context &Ctx)
-      : Constant(ID, C, Ctx) {}
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    auto ID = From->getSubclassID();
-    return ID == ClassID::ConstantVector || ID == ClassID::ConstantStruct ||
-           ID == ClassID::ConstantArray;
-  }
-};
-
-class ConstantArray final : public ConstantAggregate {
-  ConstantArray(llvm::ConstantArray *C, Context &Ctx)
-      : ConstantAggregate(ClassID::ConstantArray, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  static Constant *get(ArrayType *T, ArrayRef<Constant *> V);
-  ArrayType *getType() const;
-
-  // TODO: Missing functions: getType(), getTypeForElements(), getAnon(), get().
-
-  /// For isa/dyn_cast.
-  static bool classof(const Value *From) {
-    return From->getSubclassID() == ClassID::ConstantArray;
-  }
-};
-
-class ConstantStruct final : public ConstantAggregate {
-  ConstantStruct(llvm::ConstantStruct *C, Context &Ctx)
-      : ConstantAggregate(ClassID::ConstantStruct, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  static Constant *get(StructType *T, ArrayRef<Constant *> V);
-
-  template <typename... Csts>
-  static std::enable_if_t<are_base_of<Constant, Csts...>::value, Constant *>
-  get(StructType *T, Csts *...Vs) {
-    return get(T, ArrayRef<Constant *>({Vs...}));
-  }
-  /// Return an anonymous struct that has the specified elements.
-  /// If the struct is possibly empty, then you must specify a context.
-  static Constant *getAnon(ArrayRef<Constant *> V, bool Packed = false) {
-    return get(getTypeForElements(V, Packed), V);
-  }
-  static Constant *getAnon(Context &Ctx, ArrayRef<Constant *> V,
-                           bool Packed = false) {
-    return get(getTypeForElements(Ctx, V, Packed), V);
-  }
-  /// This version of the method allows an empty list.
-  static StructType *getTypeForElements(Context &Ctx, ArrayRef<Constant *> V,
-                                        bool Packed = false);
-  /// Return an anonymous struct type to use for a constant with the specified
-  /// set of elements. The list must not be empty.
-  static StructType *getTypeForElements(ArrayRef<Constant *> V,
-                                        bool Packed = false) {
-    assert(!V.empty() &&
-           "ConstantStruct::getTypeForElements cannot be called on empty list");
-    return getTypeForElements(V[0]->getContext(), V, Packed);
-  }
-
-  /// Specialization - reduce amount of casting.
-  inline StructType *getType() const {
-    return cast<StructType>(Value::getType());
-  }
-
-  /// For isa/dyn_cast.
-  static bool classof(const Value *From) {
-    return From->getSubclassID() == ClassID::ConstantStruct;
-  }
-};
-
-class ConstantVector final : public ConstantAggregate {
-  ConstantVector(llvm::ConstantVector *C, Context &Ctx)
-      : ConstantAggregate(ClassID::ConstantVector, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  // TODO: Missing functions: getSplat(), getType(), getSplatValue(), get().
-
-  /// For isa/dyn_cast.
-  static bool classof(const Value *From) {
-    return From->getSubclassID() == ClassID::ConstantVector;
-  }
-};
-
-// TODO: Inherit from ConstantData.
-class ConstantAggregateZero final : public Constant {
-  ConstantAggregateZero(llvm::ConstantAggregateZero *C, Context &Ctx)
-      : Constant(ClassID::ConstantAggregateZero, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  static ConstantAggregateZero *get(Type *Ty);
-  /// If this CAZ has array or vector type, return a zero with the right element
-  /// type.
-  Constant *getSequentialElement() const;
-  /// If this CAZ has struct type, return a zero with the right element type for
-  /// the specified element.
-  Constant *getStructElement(unsigned Elt) const;
-  /// Return a zero of the right value for the specified GEP index if we can,
-  /// otherwise return null (e.g. if C is a ConstantExpr).
-  Constant *getElementValue(Constant *C) const;
-  /// Return a zero of the right value for the specified GEP index.
-  Constant *getElementValue(unsigned Idx) const;
-  /// Return the number of elements in the array, vector, or struct.
-  ElementCount getElementCount() const {
-    return cast<llvm::ConstantAggregateZero>(Val)->getElementCount();
-  }
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantAggregateZero;
-  }
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("ConstantAggregateZero has no operands!");
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::ConstantAggregateZero>(Val) && "Expected a CAZ!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-// TODO: Inherit from ConstantData.
-class ConstantPointerNull final : public Constant {
-  ConstantPointerNull(llvm::ConstantPointerNull *C, Context &Ctx)
-      : Constant(ClassID::ConstantPointerNull, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  static ConstantPointerNull *get(PointerType *Ty);
-
-  PointerType *getType() const;
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantPointerNull;
-  }
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("ConstantPointerNull has no operands!");
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::ConstantPointerNull>(Val) && "Expected a CPNull!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-// TODO: Inherit from ConstantData.
-class UndefValue : public Constant {
-protected:
-  UndefValue(llvm::UndefValue *C, Context &Ctx)
-      : Constant(ClassID::UndefValue, C, Ctx) {}
-  UndefValue(ClassID ID, llvm::Constant *C, Context &Ctx)
-      : Constant(ID, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Static factory methods - Return an 'undef' object of the specified type.
-  static UndefValue *get(Type *T);
-
-  /// If this Undef has array or vector type, return a undef with the right
-  /// element type.
-  UndefValue *getSequentialElement() const;
-
-  /// If this undef has struct type, return a undef with the right element type
-  /// for the specified element.
-  UndefValue *getStructElement(unsigned Elt) const;
-
-  /// Return an undef of the right value for the specified GEP index if we can,
-  /// otherwise return null (e.g. if C is a ConstantExpr).
-  UndefValue *getElementValue(Constant *C) const;
-
-  /// Return an undef of the right value for the specified GEP index.
-  UndefValue *getElementValue(unsigned Idx) const;
-
-  /// Return the number of elements in the array, vector, or struct.
-  unsigned getNumElements() const {
-    return cast<llvm::UndefValue>(Val)->getNumElements();
-  }
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::UndefValue ||
-           From->getSubclassID() == ClassID::PoisonValue;
-  }
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("UndefValue has no operands!");
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::UndefValue>(Val) && "Expected an UndefValue!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-class PoisonValue final : public UndefValue {
-  PoisonValue(llvm::PoisonValue *C, Context &Ctx)
-      : UndefValue(ClassID::PoisonValue, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Static factory methods - Return an 'poison' object of the specified type.
-  static PoisonValue *get(Type *T);
-
-  /// If this poison has array or vector type, return a poison with the right
-  /// element type.
-  PoisonValue *getSequentialElement() const;
-
-  /// If this poison has struct type, return a poison with the right element
-  /// type for the specified element.
-  PoisonValue *getStructElement(unsigned Elt) const;
-
-  /// Return an poison of the right value for the specified GEP index if we can,
-  /// otherwise return null (e.g. if C is a ConstantExpr).
-  PoisonValue *getElementValue(Constant *C) const;
-
-  /// Return an poison of the right value for the specified GEP index.
-  PoisonValue *getElementValue(unsigned Idx) const;
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::PoisonValue;
-  }
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::PoisonValue>(Val) && "Expected a PoisonValue!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-class GlobalValue : public Constant {
-protected:
-  GlobalValue(ClassID ID, llvm::GlobalValue *C, Context &Ctx)
-      : Constant(ID, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  using LinkageTypes = llvm::GlobalValue::LinkageTypes;
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    switch (From->getSubclassID()) {
-    case ClassID::Function:
-    case ClassID::GlobalVariable:
-    case ClassID::GlobalAlias:
-    case ClassID::GlobalIFunc:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  unsigned getAddressSpace() const {
-    return cast<llvm::GlobalValue>(Val)->getAddressSpace();
-  }
-  bool hasGlobalUnnamedAddr() const {
-    return cast<llvm::GlobalValue>(Val)->hasGlobalUnnamedAddr();
-  }
-
-  /// Returns true if this value's address is not significant in this module.
-  /// This attribute is intended to be used only by the code generator and LTO
-  /// to allow the linker to decide whether the global needs to be in the symbol
-  /// table. It should probably not be used in optimizations, as the value may
-  /// have uses outside the module; use hasGlobalUnnamedAddr() instead.
-  bool hasAtLeastLocalUnnamedAddr() const {
-    return cast<llvm::GlobalValue>(Val)->hasAtLeastLocalUnnamedAddr();
-  }
-
-  using UnnamedAddr = llvm::GlobalValue::UnnamedAddr;
-
-  UnnamedAddr getUnnamedAddr() const {
-    return cast<llvm::GlobalValue>(Val)->getUnnamedAddr();
-  }
-  void setUnnamedAddr(UnnamedAddr V);
-
-  static UnnamedAddr getMinUnnamedAddr(UnnamedAddr A, UnnamedAddr B) {
-    return llvm::GlobalValue::getMinUnnamedAddr(A, B);
-  }
-
-  bool hasComdat() const { return cast<llvm::GlobalValue>(Val)->hasComdat(); }
-
-  // TODO: We need a SandboxIR Comdat if we want to implement getComdat().
-  using VisibilityTypes = llvm::GlobalValue::VisibilityTypes;
-  VisibilityTypes getVisibility() const {
-    return cast<llvm::GlobalValue>(Val)->getVisibility();
-  }
-  bool hasDefaultVisibility() const {
-    return cast<llvm::GlobalValue>(Val)->hasDefaultVisibility();
-  }
-  bool hasHiddenVisibility() const {
-    return cast<llvm::GlobalValue>(Val)->hasHiddenVisibility();
-  }
-  bool hasProtectedVisibility() const {
-    return cast<llvm::GlobalValue>(Val)->hasProtectedVisibility();
-  }
-  void setVisibility(VisibilityTypes V);
-
-  // TODO: Add missing functions.
-};
-
-class GlobalObject : public GlobalValue {
-protected:
-  GlobalObject(ClassID ID, llvm::GlobalObject *C, Context &Ctx)
-      : GlobalValue(ID, C, Ctx) {}
-  friend class Context; // For constructor.
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
-    return getOperandUseDefault(OpIdx, Verify);
-  }
-
-public:
-  unsigned getUseOperandNo(const Use &Use) const final {
-    return getUseOperandNoDefault(Use);
-  }
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    switch (From->getSubclassID()) {
-    case ClassID::Function:
-    case ClassID::GlobalVariable:
-    case ClassID::GlobalIFunc:
-      return true;
-    default:
-      return false;
-    }
-  }
-
-  /// FIXME: Remove this function once transition to Align is over.
-  uint64_t getAlignment() const {
-    return cast<llvm::GlobalObject>(Val)->getAlignment();
-  }
-
-  /// Returns the alignment of the given variable or function.
-  ///
-  /// Note that for functions this is the alignment of the code, not the
-  /// alignment of a function pointer.
-  MaybeAlign getAlign() const {
-    return cast<llvm::GlobalObject>(Val)->getAlign();
-  }
-
-  // TODO: Add missing: setAlignment(Align)
-
-  /// Sets the alignment attribute of the GlobalObject.
-  /// This method will be deprecated as the alignment property should always be
-  /// defined.
-  void setAlignment(MaybeAlign Align);
-
-  unsigned getGlobalObjectSubClassData() const {
-    return cast<llvm::GlobalObject>(Val)->getGlobalObjectSubClassData();
-  }
-
-  void setGlobalObjectSubClassData(unsigned V);
-
-  /// Check if this global has a custom object file section.
-  ///
-  /// This is more efficient than calling getSection() and checking for an empty
-  /// string.
-  bool hasSection() const {
-    return cast<llvm::GlobalObject>(Val)->hasSection();
-  }
-
-  /// Get the custom section of this global if it has one.
-  ///
-  /// If this global does not have a custom section, this will be empty and the
-  /// default object file section (.text, .data, etc) will be used.
-  StringRef getSection() const {
-    return cast<llvm::GlobalObject>(Val)->getSection();
-  }
-
-  /// Change the section for this global.
-  ///
-  /// Setting the section to the empty string tells LLVM to choose an
-  /// appropriate default object file section.
-  void setSection(StringRef S);
-
-  bool hasComdat() const { return cast<llvm::GlobalObject>(Val)->hasComdat(); }
-
-  // TODO: implement get/setComdat(), etc. once we have a sandboxir::Comdat.
-
-  // TODO: We currently don't support Metadata in sandboxir so all
-  // Metadata-related functions are missing.
-
-  using VCallVisibility = llvm::GlobalObject::VCallVisibility;
-
-  VCallVisibility getVCallVisibility() const {
-    return cast<llvm::GlobalObject>(Val)->getVCallVisibility();
-  }
-
-  /// Returns true if the alignment of the value can be unilaterally
-  /// increased.
-  ///
-  /// Note that for functions this is the alignment of the code, not the
-  /// alignment of a function pointer.
-  bool canIncreaseAlignment() const {
-    return cast<llvm::GlobalObject>(Val)->canIncreaseAlignment();
-  }
-};
-
-/// Provides API functions, like getIterator() and getReverseIterator() to
-/// GlobalIFunc, Function, GlobalVariable and GlobalAlias. In LLVM IR these are
-/// provided by ilist_node.
-template <typename GlobalT, typename LLVMGlobalT, typename ParentT,
-          typename LLVMParentT>
-class GlobalWithNodeAPI : public ParentT {
-  /// Helper for mapped_iterator.
-  struct LLVMGVToGV {
-    Context &Ctx;
-    LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
-    GlobalT &operator()(LLVMGlobalT &LLVMGV) const;
-  };
-
-public:
-  GlobalWithNodeAPI(Value::ClassID ID, LLVMParentT *C, Context &Ctx)
-      : ParentT(ID, C, Ctx) {}
-
-  // TODO: Missing getParent(). Should be added once Module is available.
-
-  using iterator = mapped_iterator<
-      decltype(static_cast<LLVMGlobalT *>(nullptr)->getIterator()), LLVMGVToGV>;
-  using reverse_iterator = mapped_iterator<
-      decltype(static_cast<LLVMGlobalT *>(nullptr)->getReverseIterator()),
-      LLVMGVToGV>;
-  iterator getIterator() const {
-    auto *LLVMGV = cast<LLVMGlobalT>(this->Val);
-    LLVMGVToGV ToGV(this->Ctx);
-    return map_iterator(LLVMGV->getIterator(), ToGV);
-  }
-  reverse_iterator getReverseIterator() const {
-    auto *LLVMGV = cast<LLVMGlobalT>(this->Val);
-    LLVMGVToGV ToGV(this->Ctx);
-    return map_iterator(LLVMGV->getReverseIterator(), ToGV);
-  }
-};
-
-class GlobalIFunc final
-    : public GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
-                               llvm::GlobalObject> {
-  GlobalIFunc(llvm::GlobalObject *C, Context &Ctx)
-      : GlobalWithNodeAPI(ClassID::GlobalIFunc, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::GlobalIFunc;
-  }
-
-  // TODO: Missing create() because we don't have a sandboxir::Module yet.
-
-  // TODO: Missing functions: copyAttributesFrom(), removeFromParent(),
-  // eraseFromParent()
-
-  void setResolver(Constant *Resolver);
-
-  Constant *getResolver() const;
-
-  // Return the resolver function after peeling off potential ConstantExpr
-  // indirection.
-  Function *getResolverFunction();
-  const Function *getResolverFunction() const {
-    return const_cast<GlobalIFunc *>(this)->getResolverFunction();
-  }
-
-  static bool isValidLinkage(LinkageTypes L) {
-    return llvm::GlobalIFunc::isValidLinkage(L);
-  }
-
-  // TODO: Missing applyAlongResolverPath().
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::GlobalIFunc>(Val) && "Expected a GlobalIFunc!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-class GlobalVariable final
-    : public GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
-                               GlobalObject, llvm::GlobalObject> {
-  GlobalVariable(llvm::GlobalObject *C, Context &Ctx)
-      : GlobalWithNodeAPI(ClassID::GlobalVariable, C, Ctx) {}
-  friend class Context; // For constructor.
-
-  /// Helper for mapped_iterator.
-  struct LLVMGVToGV {
-    Context &Ctx;
-    LLVMGVToGV(Context &Ctx) : Ctx(Ctx) {}
-    GlobalVariable &operator()(llvm::GlobalVariable &LLVMGV) const;
-  };
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::GlobalVariable;
-  }
-
-  /// Definitions have initializers, declarations don't.
-  ///
-  inline bool hasInitializer() const {
-    return cast<llvm::GlobalVariable>(Val)->hasInitializer();
-  }
-
-  /// hasDefinitiveInitializer - Whether the global variable has an initializer,
-  /// and any other instances of the global (this can happen due to weak
-  /// linkage) are guaranteed to have the same initializer.
-  ///
-  /// Note that if you want to transform a global, you must use
-  /// hasUniqueInitializer() instead, because of the *_odr linkage type.
-  ///
-  /// Example:
-  ///
-  /// @a = global SomeType* null - Initializer is both definitive and unique.
-  ///
-  /// @b = global weak SomeType* null - Initializer is neither definitive nor
-  /// unique.
-  ///
-  /// @c = global weak_odr SomeType* null - Initializer is definitive, but not
-  /// unique.
-  inline bool hasDefinitiveInitializer() const {
-    return cast<llvm::GlobalVariable>(Val)->hasDefinitiveInitializer();
-  }
-
-  /// hasUniqueInitializer - Whether the global variable has an initializer, and
-  /// any changes made to the initializer will turn up in the final executable.
-  inline bool hasUniqueInitializer() const {
-    return cast<llvm::GlobalVariable>(Val)->hasUniqueInitializer();
-  }
-
-  /// getInitializer - Return the initializer for this global variable.  It is
-  /// illegal to call this method if the global is external, because we cannot
-  /// tell what the value is initialized to!
-  ///
-  Constant *getInitializer() const;
-  /// setInitializer - Sets the initializer for this global variable, removing
-  /// any existing initializer if InitVal==NULL. The initializer must have the
-  /// type getValueType().
-  void setInitializer(Constant *InitVal);
-
-  // TODO: Add missing replaceInitializer(). Requires special tracker
-
-  /// If the value is a global constant, its value is immutable throughout the
-  /// runtime execution of the program.  Assigning a value into the constant
-  /// leads to undefined behavior.
-  ///
-  bool isConstant() const {
-    return cast<llvm::GlobalVariable>(Val)->isConstant();
-  }
-  void setConstant(bool V);
-
-  bool isExternallyInitialized() const {
-    return cast<llvm::GlobalVariable>(Val)->isExternallyInitialized();
-  }
-  void setExternallyInitialized(bool Val);
-
-  // TODO: Missing copyAttributesFrom()
-
-  // TODO: Missing removeFromParent(), eraseFromParent(), dropAllReferences()
-
-  // TODO: Missing addDebugInfo(), getDebugInfo()
-
-  // TODO: Missing attribute setter functions: addAttribute(), setAttributes().
-  //       There seems to be no removeAttribute() so we can't undo them.
-
-  /// Return true if the attribute exists.
-  bool hasAttribute(Attribute::AttrKind Kind) const {
-    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
-  }
-
-  /// Return true if the attribute exists.
-  bool hasAttribute(StringRef Kind) const {
-    return cast<llvm::GlobalVariable>(Val)->hasAttribute(Kind);
-  }
-
-  /// Return true if any attributes exist.
-  bool hasAttributes() const {
-    return cast<llvm::GlobalVariable>(Val)->hasAttributes();
-  }
-
-  /// Return the attribute object.
-  Attribute getAttribute(Attribute::AttrKind Kind) const {
-    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
-  }
-
-  /// Return the attribute object.
-  Attribute getAttribute(StringRef Kind) const {
-    return cast<llvm::GlobalVariable>(Val)->getAttribute(Kind);
-  }
-
-  /// Return the attribute set for this global
-  AttributeSet getAttributes() const {
-    return cast<llvm::GlobalVariable>(Val)->getAttributes();
-  }
-
-  /// Return attribute set as list with index.
-  /// FIXME: This may not be required once ValueEnumerators
-  /// in bitcode-writer can enumerate attribute-set.
-  AttributeList getAttributesAsList(unsigned Index) const {
-    return cast<llvm::GlobalVariable>(Val)->getAttributesAsList(Index);
-  }
-
-  /// Check if section name is present
-  bool hasImplicitSection() const {
-    return cast<llvm::GlobalVariable>(Val)->hasImplicitSection();
-  }
-
-  /// Get the custom code model raw value of this global.
-  ///
-  unsigned getCodeModelRaw() const {
-    return cast<llvm::GlobalVariable>(Val)->getCodeModelRaw();
-  }
-
-  /// Get the custom code model of this global if it has one.
-  ///
-  /// If this global does not have a custom code model, the empty instance
-  /// will be returned.
-  std::optional<CodeModel::Model> getCodeModel() const {
-    return cast<llvm::GlobalVariable>(Val)->getCodeModel();
-  }
-
-  // TODO: Missing setCodeModel(). Requires custom tracker.
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::GlobalVariable>(Val) && "Expected a GlobalVariable!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-class GlobalAlias final
-    : public GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
-                               llvm::GlobalValue> {
-  GlobalAlias(llvm::GlobalAlias *C, Context &Ctx)
-      : GlobalWithNodeAPI(ClassID::GlobalAlias, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::GlobalAlias;
-  }
-
-  // TODO: Missing create() due to unimplemented sandboxir::Module.
-
-  // TODO: Missing copyAttributresFrom().
-  // TODO: Missing removeFromParent(), eraseFromParent().
-
-  void setAliasee(Constant *Aliasee);
-  Constant *getAliasee() const;
-
-  const GlobalObject *getAliaseeObject() const;
-  GlobalObject *getAliaseeObject() {
-    return const_cast<GlobalObject *>(
-        static_cast<const GlobalAlias *>(this)->getAliaseeObject());
-  }
-
-  static bool isValidLinkage(LinkageTypes L) {
-    return llvm::GlobalAlias::isValidLinkage(L);
-  }
-};
-
-class NoCFIValue final : public Constant {
-  NoCFIValue(llvm::NoCFIValue *C, Context &Ctx)
-      : Constant(ClassID::NoCFIValue, C, Ctx) {}
-  friend class Context; // For constructor.
-
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
-    return getOperandUseDefault(OpIdx, Verify);
-  }
-
-public:
-  /// Return a NoCFIValue for the specified function.
-  static NoCFIValue *get(GlobalValue *GV);
-
-  GlobalValue *getGlobalValue() const;
-
-  /// NoCFIValue is always a pointer.
-  PointerType *getType() const;
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::NoCFIValue;
-  }
-
-  unsigned getUseOperandNo(const Use &Use) const final {
-    return getUseOperandNoDefault(Use);
-  }
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::NoCFIValue>(Val) && "Expected a NoCFIValue!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-class ConstantPtrAuth final : public Constant {
-  ConstantPtrAuth(llvm::ConstantPtrAuth *C, Context &Ctx)
-      : Constant(ClassID::ConstantPtrAuth, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Return a pointer signed with the specified parameters.
-  static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key,
-                              ConstantInt *Disc, Constant *AddrDisc);
-  /// The pointer that is signed in this ptrauth signed pointer.
-  Constant *getPointer() const;
-
-  /// The Key ID, an i32 constant.
-  ConstantInt *getKey() const;
-
-  /// The integer discriminator, an i64 constant, or 0.
-  ConstantInt *getDiscriminator() const;
-
-  /// The address discriminator if any, or the null constant.
-  /// If present, this must be a value equivalent to the storage location of
-  /// the only global-initializer user of the ptrauth signed pointer.
-  Constant *getAddrDiscriminator() const;
-
-  /// Whether there is any non-null address discriminator.
-  bool hasAddressDiscriminator() const {
-    return cast<llvm::ConstantPtrAuth>(Val)->hasAddressDiscriminator();
-  }
-
-  /// Whether the address uses a special address discriminator.
-  /// These discriminators can't be used in real pointer-auth values; they
-  /// can only be used in "prototype" values that indicate how some real
-  /// schema is supposed to be produced.
-  bool hasSpecialAddressDiscriminator(uint64_t Value) const {
-    return cast<llvm::ConstantPtrAuth>(Val)->hasSpecialAddressDiscriminator(
-        Value);
-  }
-
-  /// Check whether an authentication operation with key \p Key and (possibly
-  /// blended) discriminator \p Discriminator is known to be compatible with
-  /// this ptrauth signed pointer.
-  bool isKnownCompatibleWith(const Value *Key, const Value *Discriminator,
-                             const DataLayout &DL) const {
-    return cast<llvm::ConstantPtrAuth>(Val)->isKnownCompatibleWith(
-        Key->Val, Discriminator->Val, DL);
-  }
-
-  /// Produce a new ptrauth expression signing the given value using
-  /// the same schema as is stored in one.
-  ConstantPtrAuth *getWithSameSchema(Constant *Pointer) const;
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantPtrAuth;
-  }
-};
-
-class ConstantExpr : public Constant {
-  ConstantExpr(llvm::ConstantExpr *C, Context &Ctx)
-      : Constant(ClassID::ConstantExpr, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantExpr;
-  }
-  // TODO: Missing functions.
-};
-
-class BlockAddress final : public Constant {
-  BlockAddress(llvm::BlockAddress *C, Context &Ctx)
-      : Constant(ClassID::BlockAddress, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Return a BlockAddress for the specified function and basic block.
-  static BlockAddress *get(Function *F, BasicBlock *BB);
-
-  /// Return a BlockAddress for the specified basic block.  The basic
-  /// block must be embedded into a function.
-  static BlockAddress *get(BasicBlock *BB);
-
-  /// Lookup an existing \c BlockAddress constant for the given BasicBlock.
-  ///
-  /// \returns 0 if \c !BB->hasAddressTaken(), otherwise the \c BlockAddress.
-  static BlockAddress *lookup(const BasicBlock *BB);
-
-  Function *getFunction() const;
-  BasicBlock *getBasicBlock() const;
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::BlockAddress;
-  }
-};
-
-class DSOLocalEquivalent final : public Constant {
-  DSOLocalEquivalent(llvm::DSOLocalEquivalent *C, Context &Ctx)
-      : Constant(ClassID::DSOLocalEquivalent, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Return a DSOLocalEquivalent for the specified global value.
-  static DSOLocalEquivalent *get(GlobalValue *GV);
-
-  GlobalValue *getGlobalValue() const;
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::DSOLocalEquivalent;
-  }
-
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("DSOLocalEquivalent has no operands!");
-  }
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::DSOLocalEquivalent>(Val) &&
-           "Expected a DSOLocalEquivalent!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-// TODO: This should inherit from ConstantData.
-class ConstantTokenNone final : public Constant {
-  ConstantTokenNone(llvm::ConstantTokenNone *C, Context &Ctx)
-      : Constant(ClassID::ConstantTokenNone, C, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// Return the ConstantTokenNone.
-  static ConstantTokenNone *get(Context &Ctx);
-
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::ConstantTokenNone;
-  }
-
-  unsigned getUseOperandNo(const Use &Use) const final {
-    llvm_unreachable("ConstantTokenNone has no operands!");
-  }
-
-#ifndef NDEBUG
-  void verify() const override {
-    assert(isa<llvm::ConstantTokenNone>(Val) &&
-           "Expected a ConstantTokenNone!");
-  }
-  void dumpOS(raw_ostream &OS) const override {
-    dumpCommonPrefix(OS);
-    dumpCommonSuffix(OS);
-  }
-#endif
-};
-
-/// Iterator for `Instruction`s in a `BasicBlock.
-/// \Returns an sandboxir::Instruction & when derereferenced.
-class BBIterator {
-public:
-  using difference_type = std::ptrdiff_t;
-  using value_type = Instruction;
-  using pointer = value_type *;
-  using reference = value_type &;
-  using iterator_category = std::bidirectional_iterator_tag;
-
-private:
-  llvm::BasicBlock *BB;
-  llvm::BasicBlock::iterator It;
-  Context *Ctx;
-  pointer getInstr(llvm::BasicBlock::iterator It) const;
-
-public:
-  BBIterator() : BB(nullptr), Ctx(nullptr) {}
-  BBIterator(llvm::BasicBlock *BB, llvm::BasicBlock::iterator It, Context *Ctx)
-      : BB(BB), It(It), Ctx(Ctx) {}
-  reference operator*() const { return *getInstr(It); }
-  BBIterator &operator++();
-  BBIterator operator++(int) {
-    auto Copy = *this;
-    ++*this;
-    return Copy;
-  }
-  BBIterator &operator--();
-  BBIterator operator--(int) {
-    auto Copy = *this;
-    --*this;
-    return Copy;
-  }
-  bool operator==(const BBIterator &Other) const {
-    assert(Ctx == Other.Ctx && "BBIterators in different context!");
-    return It == Other.It;
-  }
-  bool operator!=(const BBIterator &Other) const { return !(*this == Other); }
-  /// \Returns the SBInstruction that corresponds to this iterator, or null if
-  /// the instruction is not found in the IR-to-SandboxIR tables.
-  pointer get() const { return getInstr(It); }
-  /// \Returns the parent BB.
-  BasicBlock *getNodeParent() const;
-};
 
-/// Contains a list of sandboxir::Instruction's.
-class BasicBlock : public Value {
-  /// Builds a graph that contains all values in \p BB in their original form
-  /// i.e., no vectorization is taking place here.
-  void buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB);
-  friend class Context;     // For `buildBasicBlockFromIR`
-  friend class Instruction; // For LLVM Val.
-
-  BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
-      : Value(ClassID::Block, BB, SBCtx) {
-    buildBasicBlockFromLLVMIR(BB);
-  }
+#ifndef LLVM_SANDBOXIR_INSTRUCTION_H
+#define LLVM_SANDBOXIR_INSTRUCTION_H
 
-public:
-  ~BasicBlock() = default;
-  /// For isa/dyn_cast.
-  static bool classof(const Value *From) {
-    return From->getSubclassID() == Value::ClassID::Block;
-  }
-  Function *getParent() const;
-  using iterator = BBIterator;
-  iterator begin() const;
-  iterator end() const {
-    auto *BB = cast<llvm::BasicBlock>(Val);
-    return iterator(BB, BB->end(), &Ctx);
-  }
-  std::reverse_iterator<iterator> rbegin() const {
-    return std::make_reverse_iterator(end());
-  }
-  std::reverse_iterator<iterator> rend() const {
-    return std::make_reverse_iterator(begin());
-  }
-  Context &getContext() const { return Ctx; }
-  Instruction *getTerminator() const;
-  bool empty() const { return begin() == end(); }
-  Instruction &front() const;
-  Instruction &back() const;
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/User.h"
 
-#ifndef NDEBUG
-  void verify() const final;
-  void dumpOS(raw_ostream &OS) const final;
-#endif
-};
+namespace llvm::sandboxir {
 
 /// A sandboxir::User with operands, opcode and linked with previous/next
 /// instructions in an instruction list.
-class Instruction : public sandboxir::User {
+class Instruction : public User {
 public:
   enum class Opcode {
 #define OP(OPC) OPC,
@@ -1874,7 +32,7 @@ class Instruction : public sandboxir::User {
 protected:
   Instruction(ClassID ID, Opcode Opc, llvm::Instruction *I,
               sandboxir::Context &SBCtx)
-      : sandboxir::User(ID, I, SBCtx), Opc(Opc) {}
+      : User(ID, I, SBCtx), Opc(Opc) {}
 
   Opcode Opc;
 
@@ -1937,7 +95,13 @@ class Instruction : public sandboxir::User {
   /// state to allow for new SandboxIR-specific instructions.
   Opcode getOpcode() const { return Opc; }
 
-  // TODO: Missing function getOpcodeName().
+  const char *getOpcodeName() const { return getOpcodeName(Opc); }
+
+  // Note that these functions below are calling into llvm::Instruction.
+  // A sandbox IR instruction could introduce a new opcode that could change the
+  // behavior of one of these functions. It is better that these functions are
+  // only added as needed and new sandbox IR instructions must explicitly check
+  // if any of these functions could have a different behavior.
 
   bool isTerminator() const {
     return cast<llvm::Instruction>(Val)->isTerminator();
@@ -1949,6 +113,41 @@ class Instruction : public sandboxir::User {
   }
   bool isShift() const { return cast<llvm::Instruction>(Val)->isShift(); }
   bool isCast() const { return cast<llvm::Instruction>(Val)->isCast(); }
+  bool isFuncletPad() const {
+    return cast<llvm::Instruction>(Val)->isFuncletPad();
+  }
+  bool isSpecialTerminator() const {
+    return cast<llvm::Instruction>(Val)->isSpecialTerminator();
+  }
+  bool isOnlyUserOfAnyOperand() const {
+    return cast<llvm::Instruction>(Val)->isOnlyUserOfAnyOperand();
+  }
+  bool isLogicalShift() const {
+    return cast<llvm::Instruction>(Val)->isLogicalShift();
+  }
+
+  //===--------------------------------------------------------------------===//
+  // Metadata manipulation.
+  //===--------------------------------------------------------------------===//
+
+  /// Return true if the instruction has any metadata attached to it.
+  bool hasMetadata() const {
+    return cast<llvm::Instruction>(Val)->hasMetadata();
+  }
+
+  /// Return true if this instruction has metadata attached to it other than a
+  /// debug location.
+  bool hasMetadataOtherThanDebugLoc() const {
+    return cast<llvm::Instruction>(Val)->hasMetadataOtherThanDebugLoc();
+  }
+
+  /// Return true if this instruction has the given type of metadata attached.
+  bool hasMetadata(unsigned KindID) const {
+    return cast<llvm::Instruction>(Val)->hasMetadata(KindID);
+  }
+
+  // TODO: Implement getMetadata and getAllMetadata after sandboxir::MDNode is
+  // available.
 
   // TODO: More missing functions
 
@@ -4537,237 +2736,6 @@ class OpaqueInst : public SingleLLVMInstructionImpl<llvm::Instruction> {
   }
 };
 
-class Context {
-protected:
-  LLVMContext &LLVMCtx;
-  friend class Type;        // For LLVMCtx.
-  friend class PointerType; // For LLVMCtx.
-  friend class CmpInst; // For LLVMCtx. TODO: cleanup when sandboxir::VectorType
-                        // is complete
-  friend class IntegerType;           // For LLVMCtx.
-  friend class StructType;            // For LLVMCtx.
-  friend class ::llvm::TargetExtType; // For LLVMCtx.
-  friend class Region;                // For LLVMCtx.
-
-  Tracker IRTracker;
-
-  /// Maps LLVM Value to the corresponding sandboxir::Value. Owns all
-  /// SandboxIR objects.
-  DenseMap<llvm::Value *, std::unique_ptr<sandboxir::Value>>
-      LLVMValueToValueMap;
-
-  /// Type has a protected destructor to prohibit the user from managing the
-  /// lifetime of the Type objects. Context is friend of Type, and this custom
-  /// deleter can destroy Type.
-  struct TypeDeleter {
-    void operator()(Type *Ty) { delete Ty; }
-  };
-  /// Maps LLVM Type to the corresonding sandboxir::Type. Owns all Sandbox IR
-  /// Type objects.
-  DenseMap<llvm::Type *, std::unique_ptr<Type, TypeDeleter>> LLVMTypeToTypeMap;
-
-  /// Remove \p V from the maps and returns the unique_ptr.
-  std::unique_ptr<Value> detachLLVMValue(llvm::Value *V);
-  /// Remove \p SBV from all SandboxIR maps and stop owning it. This effectively
-  /// detaches \p V from the underlying IR.
-  std::unique_ptr<Value> detach(Value *V);
-  friend void Instruction::eraseFromParent(); // For detach().
-  /// Take ownership of VPtr and store it in `LLVMValueToValueMap`.
-  Value *registerValue(std::unique_ptr<Value> &&VPtr);
-  friend class EraseFromParent; // For registerValue().
-  /// This is the actual function that creates sandboxir values for \p V,
-  /// and among others handles all instruction types.
-  Value *getOrCreateValueInternal(llvm::Value *V, llvm::User *U = nullptr);
-  /// Get or create a sandboxir::Argument for an existing LLVM IR \p LLVMArg.
-  Argument *getOrCreateArgument(llvm::Argument *LLVMArg) {
-    auto Pair = LLVMValueToValueMap.insert({LLVMArg, nullptr});
-    auto It = Pair.first;
-    if (Pair.second) {
-      It->second = std::unique_ptr<Argument>(new Argument(LLVMArg, *this));
-      return cast<Argument>(It->second.get());
-    }
-    return cast<Argument>(It->second.get());
-  }
-  /// Get or create a sandboxir::Value for an existing LLVM IR \p LLVMV.
-  Value *getOrCreateValue(llvm::Value *LLVMV) {
-    return getOrCreateValueInternal(LLVMV, 0);
-  }
-  /// Get or create a sandboxir::Constant from an existing LLVM IR \p LLVMC.
-  Constant *getOrCreateConstant(llvm::Constant *LLVMC) {
-    return cast<Constant>(getOrCreateValueInternal(LLVMC, 0));
-  }
-  // Friends for getOrCreateConstant().
-#define DEF_CONST(ID, CLASS) friend class CLASS;
-#include "llvm/SandboxIR/SandboxIRValues.def"
-
-  /// Create a sandboxir::BasicBlock for an existing LLVM IR \p BB. This will
-  /// also create all contents of the block.
-  BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
-  friend class BasicBlock; // For getOrCreateValue().
-
-  IRBuilder<ConstantFolder> LLVMIRBuilder;
-  auto &getLLVMIRBuilder() { return LLVMIRBuilder; }
-
-  VAArgInst *createVAArgInst(llvm::VAArgInst *SI);
-  friend VAArgInst; // For createVAArgInst()
-  FreezeInst *createFreezeInst(llvm::FreezeInst *SI);
-  friend FreezeInst; // For createFreezeInst()
-  FenceInst *createFenceInst(llvm::FenceInst *SI);
-  friend FenceInst; // For createFenceInst()
-  SelectInst *createSelectInst(llvm::SelectInst *SI);
-  friend SelectInst; // For createSelectInst()
-  InsertElementInst *createInsertElementInst(llvm::InsertElementInst *IEI);
-  friend InsertElementInst; // For createInsertElementInst()
-  ExtractElementInst *createExtractElementInst(llvm::ExtractElementInst *EEI);
-  friend ExtractElementInst; // For createExtractElementInst()
-  ShuffleVectorInst *createShuffleVectorInst(llvm::ShuffleVectorInst *SVI);
-  friend ShuffleVectorInst; // For createShuffleVectorInst()
-  ExtractValueInst *createExtractValueInst(llvm::ExtractValueInst *IVI);
-  friend ExtractValueInst; // For createExtractValueInst()
-  InsertValueInst *createInsertValueInst(llvm::InsertValueInst *IVI);
-  friend InsertValueInst; // For createInsertValueInst()
-  BranchInst *createBranchInst(llvm::BranchInst *I);
-  friend BranchInst; // For createBranchInst()
-  LoadInst *createLoadInst(llvm::LoadInst *LI);
-  friend LoadInst; // For createLoadInst()
-  StoreInst *createStoreInst(llvm::StoreInst *SI);
-  friend StoreInst; // For createStoreInst()
-  ReturnInst *createReturnInst(llvm::ReturnInst *I);
-  friend ReturnInst; // For createReturnInst()
-  CallInst *createCallInst(llvm::CallInst *I);
-  friend CallInst; // For createCallInst()
-  InvokeInst *createInvokeInst(llvm::InvokeInst *I);
-  friend InvokeInst; // For createInvokeInst()
-  CallBrInst *createCallBrInst(llvm::CallBrInst *I);
-  friend CallBrInst; // For createCallBrInst()
-  LandingPadInst *createLandingPadInst(llvm::LandingPadInst *I);
-  friend LandingPadInst; // For createLandingPadInst()
-  CatchPadInst *createCatchPadInst(llvm::CatchPadInst *I);
-  friend CatchPadInst; // For createCatchPadInst()
-  CleanupPadInst *createCleanupPadInst(llvm::CleanupPadInst *I);
-  friend CleanupPadInst; // For createCleanupPadInst()
-  CatchReturnInst *createCatchReturnInst(llvm::CatchReturnInst *I);
-  friend CatchReturnInst; // For createCatchReturnInst()
-  CleanupReturnInst *createCleanupReturnInst(llvm::CleanupReturnInst *I);
-  friend CleanupReturnInst; // For createCleanupReturnInst()
-  GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
-  friend GetElementPtrInst; // For createGetElementPtrInst()
-  CatchSwitchInst *createCatchSwitchInst(llvm::CatchSwitchInst *I);
-  friend CatchSwitchInst; // For createCatchSwitchInst()
-  ResumeInst *createResumeInst(llvm::ResumeInst *I);
-  friend ResumeInst; // For createResumeInst()
-  SwitchInst *createSwitchInst(llvm::SwitchInst *I);
-  friend SwitchInst; // For createSwitchInst()
-  UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
-  friend UnaryOperator; // For createUnaryOperator()
-  BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
-  friend BinaryOperator; // For createBinaryOperator()
-  AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
-  friend AtomicRMWInst; // For createAtomicRMWInst()
-  AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
-  friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
-  AllocaInst *createAllocaInst(llvm::AllocaInst *I);
-  friend AllocaInst; // For createAllocaInst()
-  CastInst *createCastInst(llvm::CastInst *I);
-  friend CastInst; // For createCastInst()
-  PHINode *createPHINode(llvm::PHINode *I);
-  friend PHINode; // For createPHINode()
-  UnreachableInst *createUnreachableInst(llvm::UnreachableInst *UI);
-  friend UnreachableInst; // For createUnreachableInst()
-  CmpInst *createCmpInst(llvm::CmpInst *I);
-  friend CmpInst; // For createCmpInst()
-  ICmpInst *createICmpInst(llvm::ICmpInst *I);
-  friend ICmpInst; // For createICmpInst()
-  FCmpInst *createFCmpInst(llvm::FCmpInst *I);
-  friend FCmpInst; // For createFCmpInst()
-
-public:
-  Context(LLVMContext &LLVMCtx)
-      : LLVMCtx(LLVMCtx), IRTracker(*this),
-        LLVMIRBuilder(LLVMCtx, ConstantFolder()) {}
-
-  Tracker &getTracker() { return IRTracker; }
-  /// Convenience function for `getTracker().save()`
-  void save() { IRTracker.save(); }
-  /// Convenience function for `getTracker().revert()`
-  void revert() { IRTracker.revert(); }
-  /// Convenience function for `getTracker().accept()`
-  void accept() { IRTracker.accept(); }
-
-  sandboxir::Value *getValue(llvm::Value *V) const;
-  const sandboxir::Value *getValue(const llvm::Value *V) const {
-    return getValue(const_cast<llvm::Value *>(V));
-  }
-
-  Type *getType(llvm::Type *LLVMTy) {
-    if (LLVMTy == nullptr)
-      return nullptr;
-    auto Pair = LLVMTypeToTypeMap.insert({LLVMTy, nullptr});
-    auto It = Pair.first;
-    if (Pair.second)
-      It->second = std::unique_ptr<Type, TypeDeleter>(new Type(LLVMTy, *this));
-    return It->second.get();
-  }
-
-  /// Create a sandboxir::Function for an existing LLVM IR \p F, including all
-  /// blocks and instructions.
-  /// This is the main API function for creating Sandbox IR.
-  Function *createFunction(llvm::Function *F);
-
-  /// \Returns the number of values registered with Context.
-  size_t getNumValues() const { return LLVMValueToValueMap.size(); }
-};
-
-class Function : public GlobalWithNodeAPI<Function, llvm::Function,
-                                          GlobalObject, llvm::GlobalObject> {
-  /// Helper for mapped_iterator.
-  struct LLVMBBToBB {
-    Context &Ctx;
-    LLVMBBToBB(Context &Ctx) : Ctx(Ctx) {}
-    BasicBlock &operator()(llvm::BasicBlock &LLVMBB) const {
-      return *cast<BasicBlock>(Ctx.getValue(&LLVMBB));
-    }
-  };
-  /// Use Context::createFunction() instead.
-  Function(llvm::Function *F, sandboxir::Context &Ctx)
-      : GlobalWithNodeAPI(ClassID::Function, F, Ctx) {}
-  friend class Context; // For constructor.
-
-public:
-  /// For isa/dyn_cast.
-  static bool classof(const sandboxir::Value *From) {
-    return From->getSubclassID() == ClassID::Function;
-  }
-
-  Argument *getArg(unsigned Idx) const {
-    llvm::Argument *Arg = cast<llvm::Function>(Val)->getArg(Idx);
-    return cast<Argument>(Ctx.getValue(Arg));
-  }
-
-  size_t arg_size() const { return cast<llvm::Function>(Val)->arg_size(); }
-  bool arg_empty() const { return cast<llvm::Function>(Val)->arg_empty(); }
-
-  using iterator = mapped_iterator<llvm::Function::iterator, LLVMBBToBB>;
-  iterator begin() const {
-    LLVMBBToBB BBGetter(Ctx);
-    return iterator(cast<llvm::Function>(Val)->begin(), BBGetter);
-  }
-  iterator end() const {
-    LLVMBBToBB BBGetter(Ctx);
-    return iterator(cast<llvm::Function>(Val)->end(), BBGetter);
-  }
-  FunctionType *getFunctionType() const;
-
-#ifndef NDEBUG
-  void verify() const final {
-    assert(isa<llvm::Function>(Val) && "Expected Function!");
-  }
-  void dumpNameAndArgs(raw_ostream &OS) const;
-  void dumpOS(raw_ostream &OS) const final;
-#endif
-};
-
-} // namespace sandboxir
-} // namespace llvm
+} // namespace llvm::sandboxir
 
-#endif // LLVM_SANDBOXIR_SANDBOXIR_H
+#endif // LLVM_SANDBOXIR_INSTRUCTION_H
diff --git a/llvm/include/llvm/SandboxIR/Module.h b/llvm/include/llvm/SandboxIR/Module.h
new file mode 100644
index 0000000000000..429bb04539bcb
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/Module.h
@@ -0,0 +1,92 @@
+//===- Module.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_MODULE_H
+#define LLVM_SANDBOXIR_MODULE_H
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/IR/Module.h"
+#include <string>
+
+namespace llvm {
+
+class DataLayout;
+
+namespace sandboxir {
+
+class Context;
+class Function;
+class GlobalVariable;
+class Type;
+class Constant;
+class GlobalAlias;
+class GlobalIFunc;
+
+/// In SandboxIR the Module is mainly used to access the list of global objects.
+class Module {
+  llvm::Module &LLVMM;
+  Context &Ctx;
+
+  Module(llvm::Module &LLVMM, Context &Ctx) : LLVMM(LLVMM), Ctx(Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  Context &getContext() const { return Ctx; }
+
+  Function *getFunction(StringRef Name) const;
+
+  const DataLayout &getDataLayout() const { return LLVMM.getDataLayout(); }
+
+  const std::string &getSourceFileName() const {
+    return LLVMM.getSourceFileName();
+  }
+
+  /// Look up the specified global variable in the module symbol table. If it
+  /// does not exist, return null. If AllowInternal is set to true, this
+  /// function will return types that have InternalLinkage. By default, these
+  /// types are not returned.
+  GlobalVariable *getGlobalVariable(StringRef Name, bool AllowInternal) const;
+  GlobalVariable *getGlobalVariable(StringRef Name) const {
+    return getGlobalVariable(Name, /*AllowInternal=*/false);
+  }
+  /// Return the global variable in the module with the specified name, of
+  /// arbitrary type. This method returns null if a global with the specified
+  /// name is not found.
+  GlobalVariable *getNamedGlobal(StringRef Name) const {
+    return getGlobalVariable(Name, true);
+  }
+
+  // TODO: missing getOrInsertGlobal().
+
+  /// Return the global alias in the module with the specified name, of
+  /// arbitrary type. This method returns null if a global with the specified
+  /// name is not found.
+  GlobalAlias *getNamedAlias(StringRef Name) const;
+
+  /// Return the global ifunc in the module with the specified name, of
+  /// arbitrary type. This method returns null if a global with the specified
+  /// name is not found.
+  GlobalIFunc *getNamedIFunc(StringRef Name) const;
+
+  // TODO: Missing removeGlobalVariable() eraseGlobalVariable(),
+  // insertGlobalVariable()
+
+  // TODO: Missing global_begin(), global_end(), globals().
+
+  // TODO: Missing many other functions.
+
+#ifndef NDEBUG
+  void dumpOS(raw_ostream &OS) const;
+  LLVM_DUMP_METHOD void dump() const;
+#endif // NDEBUG
+};
+
+} // namespace sandboxir
+} // namespace llvm
+
+#endif // LLVM_SANDBOXIR_MODULE_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Region.h b/llvm/include/llvm/SandboxIR/Region.h
similarity index 98%
rename from llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Region.h
rename to llvm/include/llvm/SandboxIR/Region.h
index 884f1324df782..67411f3fb741d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Region.h
+++ b/llvm/include/llvm/SandboxIR/Region.h
@@ -13,7 +13,7 @@
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm::sandboxir {
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 5fc43db82bd70..3e3e539a8c7c1 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -64,6 +64,9 @@ class SwitchInst;
 class ConstantInt;
 class ShuffleVectorInst;
 class CmpInst;
+class Module;
+class GlobalVariable;
+
 /// The base class for IR Change classes.
 class IRChangeBase {
 protected:
diff --git a/llvm/include/llvm/SandboxIR/Type.h b/llvm/include/llvm/SandboxIR/Type.h
index f99f80967797c..94ea4652c72c8 100644
--- a/llvm/include/llvm/SandboxIR/Type.h
+++ b/llvm/include/llvm/SandboxIR/Type.h
@@ -31,6 +31,8 @@ class IntegerType;
 class FunctionType;
 class ArrayType;
 class StructType;
+class TargetExtType;
+class Module;
 #define DEF_INSTR(ID, OPCODE, CLASS) class CLASS;
 #define DEF_CONST(ID, CLASS) class CLASS;
 #include "llvm/SandboxIR/SandboxIRValues.def"
@@ -57,6 +59,8 @@ class Type {
   friend class CmpInst;            // For LLVMTy. TODO: Cleanup after
                                    // sandboxir::VectorType is more complete.
   friend class Utils;              // for LLVMTy
+  friend class TargetExtType;      // For LLVMTy.
+  friend class Module;             // For LLVMTy.
 
   // Friend all instruction classes because `create()` functions use LLVMTy.
 #define DEF_INSTR(ID, OPCODE, CLASS) friend class CLASS;
diff --git a/llvm/include/llvm/SandboxIR/User.h b/llvm/include/llvm/SandboxIR/User.h
new file mode 100644
index 0000000000000..5e47ba5e727f4
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/User.h
@@ -0,0 +1,150 @@
+//===- User.h ---------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_USER_H
+#define LLVM_SANDBOXIR_USER_H
+
+#include "llvm/IR/User.h"
+#include "llvm/IR/Value.h"
+#include "llvm/SandboxIR/Use.h"
+#include "llvm/SandboxIR/Value.h"
+
+namespace llvm::sandboxir {
+
+class Context;
+
+/// Iterator for the `Use` edges of a User's operands.
+/// \Returns the operand `Use` when dereferenced.
+class OperandUseIterator {
+  sandboxir::Use Use;
+  /// Don't let the user create a non-empty OperandUseIterator.
+  OperandUseIterator(const class Use &Use) : Use(Use) {}
+  friend class User;                                  // For constructor
+#define DEF_INSTR(ID, OPC, CLASS) friend class CLASS; // For constructor
+#include "llvm/SandboxIR/SandboxIRValues.def"
+
+public:
+  using difference_type = std::ptrdiff_t;
+  using value_type = sandboxir::Use;
+  using pointer = value_type *;
+  using reference = value_type &;
+  using iterator_category = std::input_iterator_tag;
+
+  OperandUseIterator() = default;
+  value_type operator*() const;
+  OperandUseIterator &operator++();
+  OperandUseIterator operator++(int) {
+    auto Copy = *this;
+    this->operator++();
+    return Copy;
+  }
+  bool operator==(const OperandUseIterator &Other) const {
+    return Use == Other.Use;
+  }
+  bool operator!=(const OperandUseIterator &Other) const {
+    return !(*this == Other);
+  }
+  OperandUseIterator operator+(unsigned Num) const;
+  OperandUseIterator operator-(unsigned Num) const;
+  int operator-(const OperandUseIterator &Other) const;
+};
+
+/// A sandboxir::User has operands.
+class User : public Value {
+protected:
+  User(ClassID ID, llvm::Value *V, Context &Ctx) : Value(ID, V, Ctx) {}
+
+  /// \Returns the Use edge that corresponds to \p OpIdx.
+  /// Note: This is the default implementation that works for instructions that
+  /// match the underlying LLVM instruction. All others should use a different
+  /// implementation.
+  Use getOperandUseDefault(unsigned OpIdx, bool Verify) const;
+  /// \Returns the Use for the \p OpIdx'th operand. This is virtual to allow
+  /// instructions to deviate from the LLVM IR operands, which is a requirement
+  /// for sandboxir Instructions that consist of more than one LLVM Instruction.
+  virtual Use getOperandUseInternal(unsigned OpIdx, bool Verify) const = 0;
+  friend class OperandUseIterator; // for getOperandUseInternal()
+
+  /// The default implementation works only for single-LLVMIR-instruction
+  /// Users and only if they match exactly the LLVM instruction.
+  unsigned getUseOperandNoDefault(const Use &Use) const {
+    return Use.LLVMUse->getOperandNo();
+  }
+  /// \Returns the operand index of \p Use.
+  virtual unsigned getUseOperandNo(const Use &Use) const = 0;
+  friend unsigned Use::getOperandNo() const; // For getUseOperandNo()
+
+  void swapOperandsInternal(unsigned OpIdxA, unsigned OpIdxB) {
+    assert(OpIdxA < getNumOperands() && "OpIdxA out of bounds!");
+    assert(OpIdxB < getNumOperands() && "OpIdxB out of bounds!");
+    auto UseA = getOperandUse(OpIdxA);
+    auto UseB = getOperandUse(OpIdxB);
+    UseA.swap(UseB);
+  }
+
+#ifndef NDEBUG
+  void verifyUserOfLLVMUse(const llvm::Use &Use) const;
+#endif // NDEBUG
+
+public:
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From);
+  using op_iterator = OperandUseIterator;
+  using const_op_iterator = OperandUseIterator;
+  using op_range = iterator_range<op_iterator>;
+  using const_op_range = iterator_range<const_op_iterator>;
+
+  virtual op_iterator op_begin() {
+    assert(isa<llvm::User>(Val) && "Expect User value!");
+    return op_iterator(getOperandUseInternal(0, /*Verify=*/false));
+  }
+  virtual op_iterator op_end() {
+    assert(isa<llvm::User>(Val) && "Expect User value!");
+    return op_iterator(
+        getOperandUseInternal(getNumOperands(), /*Verify=*/false));
+  }
+  virtual const_op_iterator op_begin() const {
+    return const_cast<User *>(this)->op_begin();
+  }
+  virtual const_op_iterator op_end() const {
+    return const_cast<User *>(this)->op_end();
+  }
+
+  op_range operands() { return make_range<op_iterator>(op_begin(), op_end()); }
+  const_op_range operands() const {
+    return make_range<const_op_iterator>(op_begin(), op_end());
+  }
+  Value *getOperand(unsigned OpIdx) const { return getOperandUse(OpIdx).get(); }
+  /// \Returns the operand edge for \p OpIdx. NOTE: This should also work for
+  /// OpIdx == getNumOperands(), which is used for op_end().
+  Use getOperandUse(unsigned OpIdx) const {
+    return getOperandUseInternal(OpIdx, /*Verify=*/true);
+  }
+  virtual unsigned getNumOperands() const {
+    return isa<llvm::User>(Val) ? cast<llvm::User>(Val)->getNumOperands() : 0;
+  }
+
+  virtual void setOperand(unsigned OperandIdx, Value *Operand);
+  /// Replaces any operands that match \p FromV with \p ToV. Returns whether any
+  /// operands were replaced.
+  bool replaceUsesOfWith(Value *FromV, Value *ToV);
+
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::User>(Val) && "Expected User!");
+  }
+  void dumpCommonHeader(raw_ostream &OS) const final;
+  void dumpOS(raw_ostream &OS) const override {
+    // TODO: Remove this tmp implementation once we get the Instruction classes.
+  }
+#endif
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_USER_H
diff --git a/llvm/include/llvm/SandboxIR/Utils.h b/llvm/include/llvm/SandboxIR/Utils.h
index 4e8a175f54705..17fc837f555b8 100644
--- a/llvm/include/llvm/SandboxIR/Utils.h
+++ b/llvm/include/llvm/SandboxIR/Utils.h
@@ -12,8 +12,12 @@
 #ifndef LLVM_SANDBOXIR_UTILS_H
 #define LLVM_SANDBOXIR_UTILS_H
 
+#include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include <optional>
 
 namespace llvm::sandboxir {
 
@@ -57,6 +61,38 @@ class Utils {
   memoryLocationGetOrNone(const Instruction *I) {
     return llvm::MemoryLocation::getOrNone(cast<llvm::Instruction>(I->Val));
   }
+
+  /// \Returns the gap between the memory locations accessed by \p I0 and
+  /// \p I1 in bytes.
+  template <typename LoadOrStoreT>
+  static std::optional<int>
+  getPointerDiffInBytes(LoadOrStoreT *I0, LoadOrStoreT *I1, ScalarEvolution &SE,
+                        const DataLayout &DL) {
+    static_assert(std::is_same_v<LoadOrStoreT, LoadInst> ||
+                      std::is_same_v<LoadOrStoreT, StoreInst>,
+                  "Expected sandboxir::Load or sandboxir::Store!");
+    llvm::Value *Opnd0 = I0->getPointerOperand()->Val;
+    llvm::Value *Opnd1 = I1->getPointerOperand()->Val;
+    llvm::Value *Ptr0 = getUnderlyingObject(Opnd0);
+    llvm::Value *Ptr1 = getUnderlyingObject(Opnd1);
+    if (Ptr0 != Ptr1)
+      return false;
+    llvm::Type *ElemTy = llvm::Type::getInt8Ty(SE.getContext());
+    return getPointersDiff(ElemTy, Opnd0, ElemTy, Opnd1, DL, SE,
+                           /*StrictCheck=*/false, /*CheckType=*/false);
+  }
+
+  /// \Returns true if \p I0 accesses a memory location lower than \p I1.
+  /// Returns false if the difference cannot be determined, if the memory
+  /// locations are equal, or if I1 accesses a memory location greater than I0.
+  template <typename LoadOrStoreT>
+  static bool atLowerAddress(LoadOrStoreT *I0, LoadOrStoreT *I1,
+                             ScalarEvolution &SE, const DataLayout &DL) {
+    auto Diff = getPointerDiffInBytes(I0, I1, SE, DL);
+    if (!Diff)
+      return false;
+    return *Diff > 0;
+  }
 };
 } // namespace llvm::sandboxir
 
diff --git a/llvm/include/llvm/SandboxIR/Value.h b/llvm/include/llvm/SandboxIR/Value.h
new file mode 100644
index 0000000000000..ff7a56a45de16
--- /dev/null
+++ b/llvm/include/llvm/SandboxIR/Value.h
@@ -0,0 +1,277 @@
+//===- Value.h --------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SANDBOXIR_VALUE_H
+#define LLVM_SANDBOXIR_VALUE_H
+
+#include "llvm/IR/Value.h"
+#include "llvm/SandboxIR/Use.h"
+
+namespace llvm::sandboxir {
+
+// Forward declare all classes to avoid some MSVC build errors.
+#define DEF_INSTR(ID, OPC, CLASS) class CLASS;
+#define DEF_CONST(ID, CLASS) class CLASS;
+#define DEF_USER(ID, CLASS) class CLASS;
+#include "llvm/SandboxIR/SandboxIRValues.def"
+class Context;
+class FuncletPadInst;
+class Type;
+class GlobalValue;
+class GlobalObject;
+class Module;
+class UnaryInstruction;
+
+/// Iterator for the `Use` edges of a Value's users.
+/// \Returns a `Use` when dereferenced.
+class UserUseIterator {
+  sandboxir::Use Use;
+  /// Don't let the user create a non-empty UserUseIterator.
+  UserUseIterator(const class Use &Use) : Use(Use) {}
+  friend class Value; // For constructor
+
+public:
+  using difference_type = std::ptrdiff_t;
+  using value_type = sandboxir::Use;
+  using pointer = value_type *;
+  using reference = value_type &;
+  using iterator_category = std::input_iterator_tag;
+
+  UserUseIterator() = default;
+  value_type operator*() const { return Use; }
+  UserUseIterator &operator++();
+  bool operator==(const UserUseIterator &Other) const {
+    return Use == Other.Use;
+  }
+  bool operator!=(const UserUseIterator &Other) const {
+    return !(*this == Other);
+  }
+  const sandboxir::Use &getUse() const { return Use; }
+};
+
+/// A SandboxIR Value has users. This is the base class.
+class Value {
+public:
+  enum class ClassID : unsigned {
+#define DEF_VALUE(ID, CLASS) ID,
+#define DEF_USER(ID, CLASS) ID,
+#define DEF_CONST(ID, CLASS) ID,
+#define DEF_INSTR(ID, OPC, CLASS) ID,
+#include "llvm/SandboxIR/SandboxIRValues.def"
+  };
+
+protected:
+  static const char *getSubclassIDStr(ClassID ID) {
+    switch (ID) {
+#define DEF_VALUE(ID, CLASS)                                                   \
+  case ClassID::ID:                                                            \
+    return #ID;
+#define DEF_USER(ID, CLASS)                                                    \
+  case ClassID::ID:                                                            \
+    return #ID;
+#define DEF_CONST(ID, CLASS)                                                   \
+  case ClassID::ID:                                                            \
+    return #ID;
+#define DEF_INSTR(ID, OPC, CLASS)                                              \
+  case ClassID::ID:                                                            \
+    return #ID;
+#include "llvm/SandboxIR/SandboxIRValues.def"
+    }
+    llvm_unreachable("Unimplemented ID");
+  }
+
+  /// For isa/dyn_cast.
+  ClassID SubclassID;
+#ifndef NDEBUG
+  /// A unique ID used for forming the name (used for debugging).
+  unsigned UID;
+#endif
+  /// The LLVM Value that corresponds to this SandboxIR Value.
+  /// NOTE: Some sandboxir Instructions, like Packs, may include more than one
+  /// value and in these cases `Val` points to the last instruction in program
+  /// order.
+  llvm::Value *Val = nullptr;
+
+  friend class Context;               // For getting `Val`.
+  friend class User;                  // For getting `Val`.
+  friend class Use;                   // For getting `Val`.
+  friend class VAArgInst;             // For getting `Val`.
+  friend class FreezeInst;            // For getting `Val`.
+  friend class FenceInst;             // For getting `Val`.
+  friend class SelectInst;            // For getting `Val`.
+  friend class ExtractElementInst;    // For getting `Val`.
+  friend class InsertElementInst;     // For getting `Val`.
+  friend class ShuffleVectorInst;     // For getting `Val`.
+  friend class ExtractValueInst;      // For getting `Val`.
+  friend class InsertValueInst;       // For getting `Val`.
+  friend class BranchInst;            // For getting `Val`.
+  friend class LoadInst;              // For getting `Val`.
+  friend class StoreInst;             // For getting `Val`.
+  friend class ReturnInst;            // For getting `Val`.
+  friend class CallBase;              // For getting `Val`.
+  friend class CallInst;              // For getting `Val`.
+  friend class InvokeInst;            // For getting `Val`.
+  friend class CallBrInst;            // For getting `Val`.
+  friend class LandingPadInst;        // For getting `Val`.
+  friend class FuncletPadInst;        // For getting `Val`.
+  friend class CatchPadInst;          // For getting `Val`.
+  friend class CleanupPadInst;        // For getting `Val`.
+  friend class CatchReturnInst;       // For getting `Val`.
+  friend class GetElementPtrInst;     // For getting `Val`.
+  friend class ResumeInst;            // For getting `Val`.
+  friend class CatchSwitchInst;       // For getting `Val`.
+  friend class CleanupReturnInst;     // For getting `Val`.
+  friend class SwitchInst;            // For getting `Val`.
+  friend class UnaryOperator;         // For getting `Val`.
+  friend class BinaryOperator;        // For getting `Val`.
+  friend class AtomicRMWInst;         // For getting `Val`.
+  friend class AtomicCmpXchgInst;     // For getting `Val`.
+  friend class AllocaInst;            // For getting `Val`.
+  friend class CastInst;              // For getting `Val`.
+  friend class PHINode;               // For getting `Val`.
+  friend class UnreachableInst;       // For getting `Val`.
+  friend class CatchSwitchAddHandler; // For `Val`.
+  friend class CmpInst;               // For getting `Val`.
+  friend class ConstantArray;         // For `Val`.
+  friend class ConstantStruct;        // For `Val`.
+  friend class ConstantAggregateZero; // For `Val`.
+  friend class ConstantPointerNull;   // For `Val`.
+  friend class UndefValue;            // For `Val`.
+  friend class PoisonValue;           // For `Val`.
+  friend class BlockAddress;          // For `Val`.
+  friend class GlobalValue;           // For `Val`.
+  friend class DSOLocalEquivalent;    // For `Val`.
+  friend class GlobalObject;          // For `Val`.
+  friend class GlobalIFunc;           // For `Val`.
+  friend class GlobalVariable;        // For `Val`.
+  friend class GlobalAlias;           // For `Val`.
+  friend class NoCFIValue;            // For `Val`.
+  friend class ConstantPtrAuth;       // For `Val`.
+  friend class ConstantExpr;          // For `Val`.
+  friend class Utils;                 // For `Val`.
+  friend class Module;                // For `Val`.
+  // Region needs to manipulate metadata in the underlying LLVM Value, we don't
+  // expose metadata in sandboxir.
+  friend class Region;
+
+  /// All values point to the context.
+  Context &Ctx;
+  // This is used by eraseFromParent().
+  void clearValue() { Val = nullptr; }
+  template <typename ItTy, typename SBTy> friend class LLVMOpUserItToSBTy;
+
+  Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx);
+  /// Disable copies.
+  Value(const Value &) = delete;
+  Value &operator=(const Value &) = delete;
+
+public:
+  virtual ~Value() = default;
+  ClassID getSubclassID() const { return SubclassID; }
+
+  using use_iterator = UserUseIterator;
+  using const_use_iterator = UserUseIterator;
+
+  use_iterator use_begin();
+  const_use_iterator use_begin() const {
+    return const_cast<Value *>(this)->use_begin();
+  }
+  use_iterator use_end() { return use_iterator(Use(nullptr, nullptr, Ctx)); }
+  const_use_iterator use_end() const {
+    return const_cast<Value *>(this)->use_end();
+  }
+
+  iterator_range<use_iterator> uses() {
+    return make_range<use_iterator>(use_begin(), use_end());
+  }
+  iterator_range<const_use_iterator> uses() const {
+    return make_range<const_use_iterator>(use_begin(), use_end());
+  }
+
+  /// Helper for mapped_iterator.
+  struct UseToUser {
+    User *operator()(const Use &Use) const { return &*Use.getUser(); }
+  };
+
+  using user_iterator = mapped_iterator<sandboxir::UserUseIterator, UseToUser>;
+  using const_user_iterator = user_iterator;
+
+  user_iterator user_begin();
+  user_iterator user_end() {
+    return user_iterator(Use(nullptr, nullptr, Ctx), UseToUser());
+  }
+  const_user_iterator user_begin() const {
+    return const_cast<Value *>(this)->user_begin();
+  }
+  const_user_iterator user_end() const {
+    return const_cast<Value *>(this)->user_end();
+  }
+
+  iterator_range<user_iterator> users() {
+    return make_range<user_iterator>(user_begin(), user_end());
+  }
+  iterator_range<const_user_iterator> users() const {
+    return make_range<const_user_iterator>(user_begin(), user_end());
+  }
+  /// \Returns the number of user edges (not necessarily to unique users).
+  /// WARNING: This is a linear-time operation.
+  unsigned getNumUses() const;
+  /// Return true if this value has N uses or more.
+  /// This is logically equivalent to getNumUses() >= N.
+  /// WARNING: This can be expensive, as it is linear to the number of users.
+  bool hasNUsesOrMore(unsigned Num) const {
+    unsigned Cnt = 0;
+    for (auto It = use_begin(), ItE = use_end(); It != ItE; ++It) {
+      if (++Cnt >= Num)
+        return true;
+    }
+    return false;
+  }
+  /// Return true if this Value has exactly N uses.
+  bool hasNUses(unsigned Num) const {
+    unsigned Cnt = 0;
+    for (auto It = use_begin(), ItE = use_end(); It != ItE; ++It) {
+      if (++Cnt > Num)
+        return false;
+    }
+    return Cnt == Num;
+  }
+
+  Type *getType() const;
+
+  Context &getContext() const { return Ctx; }
+
+  void replaceUsesWithIf(Value *OtherV,
+                         llvm::function_ref<bool(const Use &)> ShouldReplace);
+  void replaceAllUsesWith(Value *Other);
+
+  /// \Returns the LLVM IR name of the bottom-most LLVM value.
+  StringRef getName() const { return Val->getName(); }
+
+#ifndef NDEBUG
+  /// Should crash if there is something wrong with the instruction.
+  virtual void verify() const = 0;
+  /// Returns the unique id in the form 'SB<number>.' like 'SB1.'
+  std::string getUid() const;
+  virtual void dumpCommonHeader(raw_ostream &OS) const;
+  void dumpCommonFooter(raw_ostream &OS) const;
+  void dumpCommonPrefix(raw_ostream &OS) const;
+  void dumpCommonSuffix(raw_ostream &OS) const;
+  void printAsOperandCommon(raw_ostream &OS) const;
+  friend raw_ostream &operator<<(raw_ostream &OS, const sandboxir::Value &V) {
+    V.dumpOS(OS);
+    return OS;
+  }
+  virtual void dumpOS(raw_ostream &OS) const = 0;
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_SANDBOXIR_VALUE_H
diff --git a/llvm/include/llvm/Support/AMDGPUAddrSpace.h b/llvm/include/llvm/Support/AMDGPUAddrSpace.h
index 4a278d0acc23b..a7533b99a8441 100644
--- a/llvm/include/llvm/Support/AMDGPUAddrSpace.h
+++ b/llvm/include/llvm/Support/AMDGPUAddrSpace.h
@@ -93,6 +93,33 @@ inline bool isExtendedGlobalAddrSpace(unsigned AS) {
          AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
          AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
 }
+
+inline bool isConstantAddressSpace(unsigned AS) {
+  switch (AS) {
+    using namespace AMDGPUAS;
+  case CONSTANT_ADDRESS:
+  case CONSTANT_ADDRESS_32BIT:
+  case CONSTANT_BUFFER_0:
+  case CONSTANT_BUFFER_1:
+  case CONSTANT_BUFFER_2:
+  case CONSTANT_BUFFER_3:
+  case CONSTANT_BUFFER_4:
+  case CONSTANT_BUFFER_5:
+  case CONSTANT_BUFFER_6:
+  case CONSTANT_BUFFER_7:
+  case CONSTANT_BUFFER_8:
+  case CONSTANT_BUFFER_9:
+  case CONSTANT_BUFFER_10:
+  case CONSTANT_BUFFER_11:
+  case CONSTANT_BUFFER_12:
+  case CONSTANT_BUFFER_13:
+  case CONSTANT_BUFFER_14:
+  case CONSTANT_BUFFER_15:
+    return true;
+  default:
+    return false;
+  }
+}
 } // end namespace AMDGPU
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def
index 9e70eb8d8fdd3..3556a253d875f 100644
--- a/llvm/include/llvm/Support/TargetOpcodes.def
+++ b/llvm/include/llvm/Support/TargetOpcodes.def
@@ -821,6 +821,9 @@ HANDLE_TARGET_OPCODE(G_FASIN)
 /// Floating point arctangent.
 HANDLE_TARGET_OPCODE(G_FATAN)
 
+/// Floating point arctangent of y/x.
+HANDLE_TARGET_OPCODE(G_FATAN2)
+
 /// Floating point hyperbolic cosine.
 HANDLE_TARGET_OPCODE(G_FCOSH)
 
diff --git a/llvm/include/llvm/TableGen/DirectiveEmitter.h b/llvm/include/llvm/TableGen/DirectiveEmitter.h
index 4ab0e889078ff..014a2bc4b8831 100644
--- a/llvm/include/llvm/TableGen/DirectiveEmitter.h
+++ b/llvm/include/llvm/TableGen/DirectiveEmitter.h
@@ -135,24 +135,24 @@ class Directive : public BaseRecord {
 public:
   Directive(const Record *Def) : BaseRecord(Def) {}
 
-  std::vector<Record *> getAllowedClauses() const {
-    return Def->getValueAsListOfDefs("allowedClauses");
+  std::vector<const Record *> getAllowedClauses() const {
+    return Def->getValueAsListOfConstDefs("allowedClauses");
   }
 
-  std::vector<Record *> getAllowedOnceClauses() const {
-    return Def->getValueAsListOfDefs("allowedOnceClauses");
+  std::vector<const Record *> getAllowedOnceClauses() const {
+    return Def->getValueAsListOfConstDefs("allowedOnceClauses");
   }
 
-  std::vector<Record *> getAllowedExclusiveClauses() const {
-    return Def->getValueAsListOfDefs("allowedExclusiveClauses");
+  std::vector<const Record *> getAllowedExclusiveClauses() const {
+    return Def->getValueAsListOfConstDefs("allowedExclusiveClauses");
   }
 
-  std::vector<Record *> getRequiredClauses() const {
-    return Def->getValueAsListOfDefs("requiredClauses");
+  std::vector<const Record *> getRequiredClauses() const {
+    return Def->getValueAsListOfConstDefs("requiredClauses");
   }
 
-  std::vector<Record *> getLeafConstructs() const {
-    return Def->getValueAsListOfDefs("leafConstructs");
+  std::vector<const Record *> getLeafConstructs() const {
+    return Def->getValueAsListOfConstDefs("leafConstructs");
   }
 
   Record *getAssociation() const { return Def->getValueAsDef("association"); }
@@ -203,8 +203,8 @@ class Clause : public BaseRecord {
     return Def->getValueAsString("enumClauseValue");
   }
 
-  std::vector<Record *> getClauseVals() const {
-    return Def->getValueAsListOfDefs("allowedClauseValues");
+  std::vector<const Record *> getClauseVals() const {
+    return Def->getValueAsListOfConstDefs("allowedClauseValues");
   }
 
   bool isValueOptional() const { return Def->getValueAsBit("isValueOptional"); }
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index f5e62dda6fd04..8b8bc9a0e9cf5 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1048,6 +1048,13 @@ def G_FATAN : GenericInstruction {
   let hasSideEffects = false;
 }
 
+// Floating point arctangent of a value.
+def G_FATAN2 : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type0:$src2);
+  let hasSideEffects = false;
+}
+
 // Floating point hyperbolic cosine of a value.
 def G_FCOSH : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 94e0fa2404d6f..88f253805ca99 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -64,9 +64,6 @@ namespace llvm {
     List,   // Get list of functions & BBs from a file. Selectively enables
             // basic block sections for a subset of basic blocks which can be
             // used to control object size bloats from creating sections.
-    Labels, // Do not use Basic Block Sections but label basic blocks.  This
-            // is useful when associating profile counts from virtual addresses
-            // to basic blocks.
     Preset, // Similar to list but the blocks are identified by passes which
             // seek to use Basic Block Sections, e.g. MachineFunctionSplitter.
             // This option cannot be set via the command line.
diff --git a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
index ef90f02f57f9f..5c6c3475ed910 100644
--- a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
+++ b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
@@ -33,7 +33,7 @@ enum CPUFeatures {
   FEAT_FP,
   FEAT_SIMD,
   FEAT_CRC,
-  FEAT_SHA1,
+  RESERVED_FEAT_SHA1, // previously used and now ABI legacy
   FEAT_SHA2,
   FEAT_SHA3,
   FEAT_AES,
diff --git a/llvm/include/llvm/Transforms/Utils/CodeLayout.h b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
index 3ba8b9137113b..c737643ee1014 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeLayout.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeLayout.h
@@ -49,12 +49,10 @@ std::vector<uint64_t> computeExtTspLayout(ArrayRef<uint64_t> NodeSizes,
 /// the given order, which is anti-correlated with the number of I-cache misses
 /// in a typical execution of the function.
 double calcExtTspScore(ArrayRef<uint64_t> Order, ArrayRef<uint64_t> NodeSizes,
-                       ArrayRef<uint64_t> NodeCounts,
                        ArrayRef<EdgeCount> EdgeCounts);
 
 /// Estimate the "quality" of the current node order in CFG.
 double calcExtTspScore(ArrayRef<uint64_t> NodeSizes,
-                       ArrayRef<uint64_t> NodeCounts,
                        ArrayRef<EdgeCount> EdgeCounts);
 
 /// Algorithm-specific params for Cache-Directed Sort. The values are tuned for
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index f5b91919a9692..dc7e484a40a45 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -224,6 +224,18 @@ class LoopVectorizationRequirements {
   Instruction *ExactFPMathInst = nullptr;
 };
 
+/// This holds details about a histogram operation -- a load -> update -> store
+/// sequence where each lane in a vector might be updating the same element as
+/// another lane.
+struct HistogramInfo {
+  LoadInst *Load;
+  Instruction *Update;
+  StoreInst *Store;
+
+  HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store)
+      : Load(Load), Update(Update), Store(Store) {}
+};
+
 /// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
 /// to what vectorization factor.
 /// This class does not look at the profitability of vectorization, only the
@@ -408,6 +420,20 @@ class LoopVectorizationLegality {
   unsigned getNumStores() const { return LAI->getNumStores(); }
   unsigned getNumLoads() const { return LAI->getNumLoads(); }
 
+  /// Returns a HistogramInfo* for the given instruction if it was determined
+  /// to be part of a load -> update -> store sequence where multiple lanes
+  /// may be working on the same memory address.
+  std::optional<const HistogramInfo *> getHistogramInfo(Instruction *I) const {
+    for (const HistogramInfo &HGram : Histograms)
+      if (HGram.Load == I || HGram.Update == I || HGram.Store == I)
+        return &HGram;
+
+    return std::nullopt;
+  }
+
+  /// Returns a list of all known histogram operations in the loop.
+  bool hasHistograms() const { return !Histograms.empty(); }
+
   PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
     return &PSE;
   }
@@ -472,6 +498,11 @@ class LoopVectorizationLegality {
   /// Returns true if the loop is vectorizable
   bool canVectorizeMemory();
 
+  /// If LAA cannot determine whether all dependences are safe, we may be able
+  /// to further analyse some IndirectUnsafe dependences and if they match a
+  /// certain pattern (like a histogram) then we may still be able to vectorize.
+  bool canVectorizeIndirectUnsafeDependences();
+
   /// Return true if we can vectorize this loop using the IF-conversion
   /// transformation.
   bool canVectorizeWithIfConvert();
@@ -593,6 +624,11 @@ class LoopVectorizationLegality {
   /// conditional assumes.
   SmallPtrSet<const Instruction *, 8> MaskedOp;
 
+  /// Contains all identified histogram operations, which are sequences of
+  /// load -> update -> store instructions where multiple lanes in a vector
+  /// may work on the same memory location.
+  SmallVector<HistogramInfo, 1> Histograms;
+
   /// BFI and PSI are used to check for profile guided size optimizations.
   BlockFrequencyInfo *BFI;
   ProfileSummaryInfo *PSI;
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
index 0120d9cf51fe9..7f6e6d11e5f53 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h
@@ -24,8 +24,8 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/SandboxIR/SandboxIR.h"
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrInterval.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
 
 namespace llvm::sandboxir {
 
@@ -85,7 +85,7 @@ class DependencyGraph {
   }
   /// Build/extend the dependency graph such that it includes \p Instrs. Returns
   /// the interval spanning \p Instrs.
-  InstrInterval extend(ArrayRef<Instruction *> Instrs);
+  Interval<Instruction> extend(ArrayRef<Instruction *> Instrs);
 #ifndef NDEBUG
   void print(raw_ostream &OS) const;
   LLVM_DUMP_METHOD void dump() const;
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/InstrInterval.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/InstrInterval.h
deleted file mode 100644
index 1343f521b29bb..0000000000000
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/InstrInterval.h
+++ /dev/null
@@ -1,124 +0,0 @@
-//===- InstrInterval.h ------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// The InstrInterval class is an interval of instructions in a block.
-// It provides an API for some basic operations on the interval, including some
-// simple set operations, like union, interseciton and others.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
-#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
-
-#include "llvm/SandboxIR/SandboxIR.h"
-#include <iterator>
-
-namespace llvm::sandboxir {
-
-/// A simple iterator for iterating the interval.
-template <typename DerefType, typename InstrIntervalType>
-class InstrIntervalIterator {
-  sandboxir::Instruction *I;
-  InstrIntervalType &R;
-
-public:
-  using difference_type = std::ptrdiff_t;
-  using value_type = sandboxir::Instruction;
-  using pointer = value_type *;
-  using reference = sandboxir::Instruction &;
-  using iterator_category = std::bidirectional_iterator_tag;
-
-  InstrIntervalIterator(sandboxir::Instruction *I, InstrIntervalType &R)
-      : I(I), R(R) {}
-  bool operator==(const InstrIntervalIterator &Other) const {
-    assert(&R == &Other.R && "Iterators belong to different regions!");
-    return Other.I == I;
-  }
-  bool operator!=(const InstrIntervalIterator &Other) const {
-    return !(*this == Other);
-  }
-  InstrIntervalIterator &operator++() {
-    assert(I != nullptr && "already at end()!");
-    I = I->getNextNode();
-    return *this;
-  }
-  InstrIntervalIterator operator++(int) {
-    auto ItCopy = *this;
-    ++*this;
-    return ItCopy;
-  }
-  InstrIntervalIterator &operator--() {
-    // `I` is nullptr for end() when ToI is the BB terminator.
-    I = I != nullptr ? I->getPrevNode() : R.ToI;
-    return *this;
-  }
-  InstrIntervalIterator operator--(int) {
-    auto ItCopy = *this;
-    --*this;
-    return ItCopy;
-  }
-  template <typename T =
-                std::enable_if<std::is_same<DerefType, Instruction *&>::value>>
-  sandboxir::Instruction &operator*() {
-    return *I;
-  }
-  DerefType operator*() const { return *I; }
-};
-
-class InstrInterval {
-  Instruction *FromI;
-  Instruction *ToI;
-
-public:
-  InstrInterval() : FromI(nullptr), ToI(nullptr) {}
-  InstrInterval(Instruction *FromI, Instruction *ToI) : FromI(FromI), ToI(ToI) {
-    assert((FromI == ToI || FromI->comesBefore(ToI)) &&
-           "FromI should come before TopI!");
-  }
-  InstrInterval(ArrayRef<Instruction *> Instrs) {
-    assert(!Instrs.empty() && "Expected non-empty Instrs!");
-    FromI = Instrs[0];
-    ToI = Instrs[0];
-    for (auto *I : drop_begin(Instrs)) {
-      if (I->comesBefore(FromI))
-        FromI = I;
-      else if (ToI->comesBefore(I))
-        ToI = I;
-    }
-  }
-  bool empty() const {
-    assert(((FromI == nullptr && ToI == nullptr) ||
-            (FromI != nullptr && ToI != nullptr)) &&
-           "Either none or both should be null");
-    return FromI == nullptr;
-  }
-  bool contains(Instruction *I) const {
-    if (empty())
-      return false;
-    return (FromI == I || FromI->comesBefore(I)) &&
-           (I == ToI || I->comesBefore(ToI));
-  }
-  Instruction *top() const { return FromI; }
-  Instruction *bottom() const { return ToI; }
-
-  using iterator =
-      InstrIntervalIterator<sandboxir::Instruction &, InstrInterval>;
-  using const_iterator = InstrIntervalIterator<const sandboxir::Instruction &,
-                                               const InstrInterval>;
-  iterator begin() { return iterator(FromI, *this); }
-  iterator end() {
-    return iterator(ToI != nullptr ? ToI->getNextNode() : nullptr, *this);
-  }
-  const_iterator begin() const { return const_iterator(FromI, *this); }
-  const_iterator end() const {
-    return const_iterator(ToI != nullptr ? ToI->getNextNode() : nullptr, *this);
-  }
-};
-} // namespace llvm::sandboxir
-
-#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
new file mode 100644
index 0000000000000..d088c6c556f3a
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h
@@ -0,0 +1,125 @@
+//===- Interval.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The Interval class is a generic interval of ordered objects that implement:
+// - T * T::getPrevNode()
+// - T * T::getNextNode()
+// - bool T::comesBefore(const T *) const
+//
+// This is currently used for Instruction intervals.
+// It provides an API for some basic operations on the interval, including some
+// simple set operations, like union, interseciton and others.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
+#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include <iterator>
+
+namespace llvm::sandboxir {
+
+/// A simple iterator for iterating the interval.
+template <typename T, typename IntervalType> class IntervalIterator {
+  T *I;
+  IntervalType &R;
+
+public:
+  using difference_type = std::ptrdiff_t;
+  using value_type = T;
+  using pointer = value_type *;
+  using reference = T &;
+  using iterator_category = std::bidirectional_iterator_tag;
+
+  IntervalIterator(T *I, IntervalType &R) : I(I), R(R) {}
+  bool operator==(const IntervalIterator &Other) const {
+    assert(&R == &Other.R && "Iterators belong to different regions!");
+    return Other.I == I;
+  }
+  bool operator!=(const IntervalIterator &Other) const {
+    return !(*this == Other);
+  }
+  IntervalIterator &operator++() {
+    assert(I != nullptr && "already at end()!");
+    I = I->getNextNode();
+    return *this;
+  }
+  IntervalIterator operator++(int) {
+    auto ItCopy = *this;
+    ++*this;
+    return ItCopy;
+  }
+  IntervalIterator &operator--() {
+    // `I` is nullptr for end() when To is the BB terminator.
+    I = I != nullptr ? I->getPrevNode() : R.To;
+    return *this;
+  }
+  IntervalIterator operator--(int) {
+    auto ItCopy = *this;
+    --*this;
+    return ItCopy;
+  }
+  template <typename HT = std::enable_if<std::is_same<T, T *&>::value>>
+  T &operator*() {
+    return *I;
+  }
+  T &operator*() const { return *I; }
+};
+
+template <typename T> class Interval {
+  T *From;
+  T *To;
+
+public:
+  Interval() : From(nullptr), To(nullptr) {}
+  Interval(T *From, T *To) : From(From), To(To) {
+    assert((From == To || From->comesBefore(To)) &&
+           "From should come before From!");
+  }
+  Interval(ArrayRef<T *> Elems) {
+    assert(!Elems.empty() && "Expected non-empty Elems!");
+    From = Elems[0];
+    To = Elems[0];
+    for (auto *I : drop_begin(Elems)) {
+      if (I->comesBefore(From))
+        From = I;
+      else if (To->comesBefore(I))
+        To = I;
+    }
+  }
+  bool empty() const {
+    assert(((From == nullptr && To == nullptr) ||
+            (From != nullptr && To != nullptr)) &&
+           "Either none or both should be null");
+    return From == nullptr;
+  }
+  bool contains(T *I) const {
+    if (empty())
+      return false;
+    return (From == I || From->comesBefore(I)) &&
+           (I == To || I->comesBefore(To));
+  }
+  T *top() const { return From; }
+  T *bottom() const { return To; }
+
+  using iterator = IntervalIterator<T, Interval>;
+  using const_iterator = IntervalIterator<const T, const Interval>;
+  iterator begin() { return iterator(From, *this); }
+  iterator end() {
+    return iterator(To != nullptr ? To->getNextNode() : nullptr, *this);
+  }
+  const_iterator begin() const { return const_iterator(From, *this); }
+  const_iterator end() const {
+    return const_iterator(To != nullptr ? To->getNextNode() : nullptr, *this);
+  }
+};
+
+} // namespace llvm::sandboxir
+
+#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_INSTRINTERVAL_H
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
index 78c1c0e4c0464..50fa56c5b2194 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h
@@ -12,11 +12,12 @@
 #ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_LEGALITY_H
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_LEGALITY_H
 
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/ADT/ArrayRef.h"
 
 namespace llvm::sandboxir {
 
 class LegalityAnalysis;
+class Value;
 
 enum class LegalityResultID {
   Widen, ///> Vectorize by combining scalars to a vector.
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
index 99582e3e0e023..a2108f07c28e5 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h
@@ -13,8 +13,8 @@
 #define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_PASSES_BOTTOMUPVEC_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/SandboxIR/Constant.h"
 #include "llvm/SandboxIR/Pass.h"
-#include "llvm/SandboxIR/SandboxIR.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
 
 namespace llvm::sandboxir {
diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp
index 7517011395a7d..825e517cd09f5 100644
--- a/llvm/lib/Analysis/CtxProfAnalysis.cpp
+++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp
@@ -184,6 +184,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M,
   // If we made it this far, the Result is valid - which we mark by setting
   // .Profiles.
   Result.Profiles = std::move(*MaybeCtx);
+  Result.initIndex();
   return Result;
 }
 
@@ -265,11 +266,9 @@ CtxProfAnalysis::getSelectInstrumentation(SelectInst &SI) {
 
 template <class ProfilesTy, class ProfTy>
 static void preorderVisit(ProfilesTy &Profiles,
-                          function_ref<void(ProfTy &)> Visitor,
-                          GlobalValue::GUID Match = 0) {
+                          function_ref<void(ProfTy &)> Visitor) {
   std::function<void(ProfTy &)> Traverser = [&](auto &Ctx) {
-    if (!Match || Ctx.guid() == Match)
-      Visitor(Ctx);
+    Visitor(Ctx);
     for (auto &[_, SubCtxSet] : Ctx.callsites())
       for (auto &[__, Subctx] : SubCtxSet)
         Traverser(Subctx);
@@ -278,16 +277,44 @@ static void preorderVisit(ProfilesTy &Profiles,
     Traverser(P);
 }
 
-void PGOContextualProfile::update(Visitor V, const Function *F) {
-  GlobalValue::GUID G = F ? getDefinedFunctionGUID(*F) : 0U;
+void PGOContextualProfile::initIndex() {
+  // Initialize the head of the index list for each function. We don't need it
+  // after this point.
+  DenseMap<GlobalValue::GUID, PGOCtxProfContext *> InsertionPoints;
+  for (auto &[Guid, FI] : FuncInfo)
+    InsertionPoints[Guid] = &FI.Index;
   preorderVisit<PGOCtxProfContext::CallTargetMapTy, PGOCtxProfContext>(
-      *Profiles, V, G);
+      *Profiles, [&](PGOCtxProfContext &Ctx) {
+        auto InsertIt = InsertionPoints.find(Ctx.guid());
+        if (InsertIt == InsertionPoints.end())
+          return;
+        // Insert at the end of the list. Since we traverse in preorder, it
+        // means that when we iterate the list from the beginning, we'd
+        // encounter the contexts in the order we would have, should we have
+        // performed a full preorder traversal.
+        InsertIt->second->Next = &Ctx;
+        Ctx.Previous = InsertIt->second;
+        InsertIt->second = &Ctx;
+      });
+}
+
+void PGOContextualProfile::update(Visitor V, const Function &F) {
+  assert(isFunctionKnown(F));
+  GlobalValue::GUID G = getDefinedFunctionGUID(F);
+  for (auto *Node = FuncInfo.find(G)->second.Index.Next; Node;
+       Node = Node->Next)
+    V(*reinterpret_cast<PGOCtxProfContext *>(Node));
 }
 
 void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const {
-  GlobalValue::GUID G = F ? getDefinedFunctionGUID(*F) : 0U;
-  preorderVisit<const PGOCtxProfContext::CallTargetMapTy,
-                const PGOCtxProfContext>(*Profiles, V, G);
+  if (!F)
+    return preorderVisit<const PGOCtxProfContext::CallTargetMapTy,
+                         const PGOCtxProfContext>(*Profiles, V);
+  assert(isFunctionKnown(*F));
+  GlobalValue::GUID G = getDefinedFunctionGUID(*F);
+  for (const auto *Node = FuncInfo.find(G)->second.Index.Next; Node;
+       Node = Node->Next)
+    V(*reinterpret_cast<const PGOCtxProfContext *>(Node));
 }
 
 const CtxProfFlatProfile PGOContextualProfile::flatten() const {
@@ -309,3 +336,25 @@ const CtxProfFlatProfile PGOContextualProfile::flatten() const {
       });
   return Flat;
 }
+
+void CtxProfAnalysis::collectIndirectCallPromotionList(
+    CallBase &IC, Result &Profile,
+    SetVector<std::pair<CallBase *, Function *>> &Candidates) {
+  const auto *Instr = CtxProfAnalysis::getCallsiteInstrumentation(IC);
+  if (!Instr)
+    return;
+  Module &M = *IC.getParent()->getModule();
+  const uint32_t CallID = Instr->getIndex()->getZExtValue();
+  Profile.visit(
+      [&](const PGOCtxProfContext &Ctx) {
+        const auto &Targets = Ctx.callsites().find(CallID);
+        if (Targets == Ctx.callsites().end())
+          return;
+        for (const auto &[Guid, _] : Targets->second)
+          if (auto Name = Profile.getFunctionName(Guid); !Name.empty())
+            if (auto *Target = M.getFunction(Name))
+              if (Target->hasFnAttribute(Attribute::AlwaysInline))
+                Candidates.insert({&IC, Target});
+      },
+      IC.getCaller());
+}
diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index e0a029802bbd9..4689451243cd9 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -67,6 +67,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -102,6 +103,8 @@ class Lint : public InstVisitor<Lint> {
   void visitReturnInst(ReturnInst &I);
   void visitLoadInst(LoadInst &I);
   void visitStoreInst(StoreInst &I);
+  void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+  void visitAtomicRMWInst(AtomicRMWInst &I);
   void visitXor(BinaryOperator &I);
   void visitSub(BinaryOperator &I);
   void visitLShr(BinaryOperator &I);
@@ -124,6 +127,7 @@ class Lint : public InstVisitor<Lint> {
 
 public:
   Module *Mod;
+  Triple TT;
   const DataLayout *DL;
   AliasAnalysis *AA;
   AssumptionCache *AC;
@@ -135,8 +139,8 @@ class Lint : public InstVisitor<Lint> {
 
   Lint(Module *Mod, const DataLayout *DL, AliasAnalysis *AA,
        AssumptionCache *AC, DominatorTree *DT, TargetLibraryInfo *TLI)
-      : Mod(Mod), DL(DL), AA(AA), AC(AC), DT(DT), TLI(TLI),
-        MessagesStr(Messages) {}
+      : Mod(Mod), TT(Triple::normalize(Mod->getTargetTriple())), DL(DL), AA(AA),
+        AC(AC), DT(DT), TLI(TLI), MessagesStr(Messages) {}
 
   void WriteValues(ArrayRef<const Value *> Vs) {
     for (const Value *V : Vs) {
@@ -401,6 +405,11 @@ void Lint::visitMemoryReference(Instruction &I, const MemoryLocation &Loc,
         "Unusual: Address one pointer dereference", &I);
 
   if (Flags & MemRef::Write) {
+    if (TT.isAMDGPU())
+      Check(!AMDGPU::isConstantAddressSpace(
+                UnderlyingObject->getType()->getPointerAddressSpace()),
+            "Undefined behavior: Write to memory in const addrspace", &I);
+
     if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(UnderlyingObject))
       Check(!GV->isConstant(), "Undefined behavior: Write to read-only memory",
             &I);
@@ -480,6 +489,16 @@ void Lint::visitStoreInst(StoreInst &I) {
                        I.getOperand(0)->getType(), MemRef::Write);
 }
 
+void Lint::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
+  visitMemoryReference(I, MemoryLocation::get(&I), I.getAlign(),
+                       I.getOperand(0)->getType(), MemRef::Write);
+}
+
+void Lint::visitAtomicRMWInst(AtomicRMWInst &I) {
+  visitMemoryReference(I, MemoryLocation::get(&I), I.getAlign(),
+                       I.getOperand(0)->getType(), MemRef::Write);
+}
+
 void Lint::visitXor(BinaryOperator &I) {
   Check(!isa<UndefValue>(I.getOperand(0)) || !isa<UndefValue>(I.getOperand(1)),
         "Undefined result: xor(undef, undef)", &I);
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 79504ca7b73c8..c5fba184cd085 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -888,7 +888,7 @@ void MemoryDependenceResults::getNonLocalPointerDependency(
   // each block.  Because of critical edges, we currently bail out if querying
   // a block with multiple different pointers.  This can happen during PHI
   // translation.
-  DenseMap<BasicBlock *, Value *> Visited;
+  SmallDenseMap<BasicBlock *, Value *, 16> Visited;
   if (getNonLocalPointerDepFromBB(QueryInst, Address, Loc, isLoad, FromBB,
                                    Result, Visited, true))
     return;
@@ -1038,7 +1038,7 @@ bool MemoryDependenceResults::getNonLocalPointerDepFromBB(
     Instruction *QueryInst, const PHITransAddr &Pointer,
     const MemoryLocation &Loc, bool isLoad, BasicBlock *StartBB,
     SmallVectorImpl<NonLocalDepResult> &Result,
-    DenseMap<BasicBlock *, Value *> &Visited, bool SkipFirstBlock,
+    SmallDenseMap<BasicBlock *, Value *, 16> &Visited, bool SkipFirstBlock,
     bool IsIncomplete) {
   // Look up the cached info for Pointer.
   ValueIsLoadPair CacheKey(Pointer.getAddr(), isLoad);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 233f8edca5b13..09e5c080c19cf 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -2255,7 +2255,7 @@ const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
 /// the common case where no interesting opportunities are present, and
 /// is also used as a check to avoid infinite recursion.
 static bool
-CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
+CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
                              SmallVectorImpl<const SCEV *> &NewOps,
                              APInt &AccumulatedConstant,
                              ArrayRef<const SCEV *> Ops, const APInt &Scale,
@@ -2753,7 +2753,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   // operands multiplied by constant values.
   if (Idx < Ops.size() && isa<SCEVMulExpr>(Ops[Idx])) {
     uint64_t BitWidth = getTypeSizeInBits(Ty);
-    DenseMap<const SCEV *, APInt> M;
+    SmallDenseMap<const SCEV *, APInt, 16> M;
     SmallVector<const SCEV *, 8> NewOps;
     APInt AccumulatedConstant(BitWidth, 0);
     if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 08b917fdb260a..e088c312c7b44 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -5239,7 +5239,7 @@ bool LLParser::parseDISubrange(MDNode *&Result, bool IsDistinct) {
   Metadata *UpperBound = nullptr;
   Metadata *Stride = nullptr;
 
-  auto convToMetadata = [&](MDSignedOrMDField Bound) -> Metadata * {
+  auto convToMetadata = [&](const MDSignedOrMDField &Bound) -> Metadata * {
     if (Bound.isMDSignedField())
       return ConstantAsMetadata::get(ConstantInt::getSigned(
           Type::getInt64Ty(Context), Bound.getMDSignedValue()));
@@ -5271,7 +5271,7 @@ bool LLParser::parseDIGenericSubrange(MDNode *&Result, bool IsDistinct) {
   PARSE_MD_FIELDS();
 #undef VISIT_MD_FIELDS
 
-  auto ConvToMetadata = [&](MDSignedOrMDField Bound) -> Metadata * {
+  auto ConvToMetadata = [&](const MDSignedOrMDField &Bound) -> Metadata * {
     if (Bound.isMDSignedField())
       return DIExpression::get(
           Context, {dwarf::DW_OP_consts,
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index d17800d375b7f..317278911b28f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1432,7 +1432,7 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
       OutStreamer->AddComment("BB id");
       // Emit the BB ID for this basic block.
       // We only emit BaseID since CloneID is unset for
-      // basic-block-sections=labels.
+      // -basic-block-adress-map.
       // TODO: Emit the full BBID when labels and sections can be mixed
       // together.
       OutStreamer->emitULEB128IntValue(MBB.getBBID()->BaseID);
@@ -1866,7 +1866,7 @@ void AsmPrinter::emitFunctionBody() {
     // We must emit temporary symbol for the end of this basic block, if either
     // we have BBLabels enabled or if this basic blocks marks the end of a
     // section.
-    if (MF->hasBBLabels() || MF->getTarget().Options.BBAddrMap ||
+    if (MF->getTarget().Options.BBAddrMap ||
         (MAI->hasDotTypeDotSizeDirective() && MBB.isEndSection()))
       OutStreamer->emitLabel(MBB.getEndSymbol());
 
@@ -2021,7 +2021,7 @@ void AsmPrinter::emitFunctionBody() {
   // Emit section containing BB address offsets and their metadata, when
   // BB labels are requested for this function. Skip empty functions.
   if (HasAnyRealCode) {
-    if (MF->hasBBLabels() || MF->getTarget().Options.BBAddrMap)
+    if (MF->getTarget().Options.BBAddrMap)
       emitBBAddrMapSection(*MF);
     else if (PgoAnalysisMapFeatures.getBits() != 0)
       MF->getContext().reportWarning(
@@ -2620,7 +2620,7 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
       F.hasFnAttribute("xray-instruction-threshold") ||
       needFuncLabels(MF, *this) || NeedsLocalForSize ||
       MF.getTarget().Options.EmitStackSizeSection ||
-      MF.getTarget().Options.BBAddrMap || MF.hasBBLabels()) {
+      MF.getTarget().Options.BBAddrMap) {
     CurrentFnBegin = createTempSymbol("func_begin");
     if (NeedsLocalForSize)
       CurrentFnSymForSize = CurrentFnBegin;
@@ -4155,8 +4155,7 @@ bool AsmPrinter::shouldEmitLabelForBasicBlock(
   // With `-fbasic-block-sections=`, a label is needed for every non-entry block
   // in the labels mode (option `=labels`) and every section beginning in the
   // sections mode (`=all` and `=list=`).
-  if ((MF->hasBBLabels() || MF->getTarget().Options.BBAddrMap ||
-       MBB.isBeginSection()) &&
+  if ((MF->getTarget().Options.BBAddrMap || MBB.isBeginSection()) &&
       !MBB.isEntryBlock())
     return true;
   // A label is needed for any block with at least one predecessor (when that
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 0a1ff189bedbc..20ee50dca499f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -779,6 +779,13 @@ DIE *DwarfCompileUnit::constructVariableDIE(DbgVariable &DV, bool Abstract) {
 void DwarfCompileUnit::applyConcreteDbgVariableAttributes(
     const Loc::Single &Single, const DbgVariable &DV, DIE &VariableDie) {
   const DbgValueLoc *DVal = &Single.getValueLoc();
+  if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB() &&
+      !Single.getExpr()) {
+    // Lack of expression means it is a register.  Registers for PTX need to
+    // be marked with DW_AT_address_class = 2.  See
+    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
+    addUInt(VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1, 2);
+  }
   if (!DVal->isVariadic()) {
     const DbgValueLocEntry *Entry = DVal->getLocEntries().begin();
     if (Entry->isLocation()) {
@@ -1209,7 +1216,7 @@ void DwarfCompileUnit::constructAbstractSubprogramScopeDIE(
 }
 
 bool DwarfCompileUnit::useGNUAnalogForDwarf5Feature() const {
-  return DD->getDwarfVersion() == 4 && !DD->tuneForLLDB();
+  return DD->getDwarfVersion() <= 4 && !DD->tuneForLLDB();
 }
 
 dwarf::Tag DwarfCompileUnit::getDwarf5OrGNUTag(dwarf::Tag Tag) const {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index e9649f9ff8165..6f211abb299e7 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -570,7 +570,7 @@ void DwarfDebug::constructAbstractSubprogramScopeDIE(DwarfCompileUnit &SrcCU,
 /// debug expression to a register in the forwarded register worklist.
 struct FwdRegParamInfo {
   /// The described parameter register.
-  unsigned ParamReg;
+  uint64_t ParamReg;
 
   /// Debug expression that has been built up when walking through the
   /// instruction chain that produces the parameter's value.
@@ -578,7 +578,7 @@ struct FwdRegParamInfo {
 };
 
 /// Register worklist for finding call site values.
-using FwdRegWorklist = MapVector<unsigned, SmallVector<FwdRegParamInfo, 2>>;
+using FwdRegWorklist = MapVector<uint64_t, SmallVector<FwdRegParamInfo, 2>>;
 /// Container for the set of registers known to be clobbered on the path to a
 /// call site.
 using ClobberedRegSet = SmallSet<Register, 16>;
@@ -3682,8 +3682,10 @@ void DwarfDebug::beginCodeAlignment(const MachineBasicBlock &MBB) {
     return;
 
   auto PrevLoc = Asm->OutStreamer->getContext().getCurrentDwarfLoc();
-  Asm->OutStreamer->emitDwarfLocDirective(
-      PrevLoc.getFileNum(), 0, PrevLoc.getColumn(), 0, 0, 0, StringRef());
-  MCDwarfLineEntry::make(Asm->OutStreamer.get(),
-                         Asm->OutStreamer->getCurrentSectionOnly());
+  if (PrevLoc.getLine()) {
+    Asm->OutStreamer->emitDwarfLocDirective(
+        PrevLoc.getFileNum(), 0, PrevLoc.getColumn(), 0, 0, 0, StringRef());
+    MCDwarfLineEntry::make(Asm->OutStreamer.get(),
+                           Asm->OutStreamer->getCurrentSectionOnly());
+  }
 }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 9d6e1bb367bc8..f5d2863ae70b7 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -40,7 +40,7 @@ void DwarfExpression::emitConstu(uint64_t Value) {
   }
 }
 
-void DwarfExpression::addReg(int DwarfReg, const char *Comment) {
+void DwarfExpression::addReg(int64_t DwarfReg, const char *Comment) {
   assert(DwarfReg >= 0 && "invalid negative dwarf register number");
   assert((isUnknownLocation() || isRegisterLocation()) &&
          "location description already locked down");
@@ -53,7 +53,7 @@ void DwarfExpression::addReg(int DwarfReg, const char *Comment) {
   }
 }
 
-void DwarfExpression::addBReg(int DwarfReg, int Offset) {
+void DwarfExpression::addBReg(int64_t DwarfReg, int64_t Offset) {
   assert(DwarfReg >= 0 && "invalid negative dwarf register number");
   assert(!isRegisterLocation() && "location description already locked down");
   if (DwarfReg < 32) {
@@ -65,7 +65,7 @@ void DwarfExpression::addBReg(int DwarfReg, int Offset) {
   emitSigned(Offset);
 }
 
-void DwarfExpression::addFBReg(int Offset) {
+void DwarfExpression::addFBReg(int64_t Offset) {
   emitOp(dwarf::DW_OP_fbreg);
   emitSigned(Offset);
 }
@@ -105,10 +105,16 @@ bool DwarfExpression::addMachineReg(const TargetRegisterInfo &TRI,
       DwarfRegs.push_back(Register::createRegister(-1, nullptr));
       return true;
     }
+    // Try getting dwarf register for virtual register anyway, eg. for NVPTX.
+    int64_t Reg = TRI.getDwarfRegNum(MachineReg, false);
+    if (Reg > 0) {
+      DwarfRegs.push_back(Register::createRegister(Reg, nullptr));
+      return true;
+    }
     return false;
   }
 
-  int Reg = TRI.getDwarfRegNum(MachineReg, false);
+  int64_t Reg = TRI.getDwarfRegNum(MachineReg, false);
 
   // If this is a valid register number, emit it.
   if (Reg >= 0) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
index 4daa78b15b8e2..06809ab263875 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -45,17 +45,17 @@ class DwarfExpression {
 protected:
   /// Holds information about all subregisters comprising a register location.
   struct Register {
-    int DwarfRegNo;
+    int64_t DwarfRegNo;
     unsigned SubRegSize;
     const char *Comment;
 
     /// Create a full register, no extra DW_OP_piece operators necessary.
-    static Register createRegister(int RegNo, const char *Comment) {
+    static Register createRegister(int64_t RegNo, const char *Comment) {
       return {RegNo, 0, Comment};
     }
 
     /// Create a subregister that needs a DW_OP_piece operator with SizeInBits.
-    static Register createSubRegister(int RegNo, unsigned SizeInBits,
+    static Register createSubRegister(int64_t RegNo, unsigned SizeInBits,
                                       const char *Comment) {
       return {RegNo, SizeInBits, Comment};
     }
@@ -161,13 +161,13 @@ class DwarfExpression {
 
   /// Emit a DW_OP_reg operation. Note that this is only legal inside a DWARF
   /// register location description.
-  void addReg(int DwarfReg, const char *Comment = nullptr);
+  void addReg(int64_t DwarfReg, const char *Comment = nullptr);
 
   /// Emit a DW_OP_breg operation.
-  void addBReg(int DwarfReg, int Offset);
+  void addBReg(int64_t DwarfReg, int64_t Offset);
 
   /// Emit DW_OP_fbreg <Offset>.
-  void addFBReg(int Offset);
+  void addFBReg(int64_t Offset);
 
   /// Emit a partial DWARF register operation.
   ///
diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp
index 0071284c86209..1eedfc4b25912 100644
--- a/llvm/lib/CodeGen/BasicBlockSections.cpp
+++ b/llvm/lib/CodeGen/BasicBlockSections.cpp
@@ -311,11 +311,6 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
   // original layout positions and finding the original fallthroughs.
   MF.RenumberBlocks();
 
-  if (BBSectionsType == BasicBlockSection::Labels) {
-    MF.setBBSectionsType(BBSectionsType);
-    return true;
-  }
-
   DenseMap<UniqueBBID, BBClusterInfo> FuncClusterInfo;
   if (BBSectionsType == BasicBlockSection::List) {
     auto [HasProfile, ClusterInfo] =
@@ -382,8 +377,6 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) {
 // avoids the need to store basic block IDs in the BB address map section, since
 // they can be determined implicitly.
 bool BasicBlockSections::handleBBAddrMap(MachineFunction &MF) {
-  if (MF.getTarget().getBBSectionsType() == BasicBlockSection::Labels)
-    return false;
   if (!MF.getTarget().Options.BBAddrMap)
     return false;
   MF.RenumberBlocks();
diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp
index f50eb5e1730a3..a762aab43ddd2 100644
--- a/llvm/lib/CodeGen/BranchRelaxation.cpp
+++ b/llvm/lib/CodeGen/BranchRelaxation.cpp
@@ -72,8 +72,8 @@ class BranchRelaxation : public MachineFunctionPass {
       if (Alignment <= ParentAlign)
         return alignTo(PO, Alignment);
 
-      // The alignment of this MBB is larger than the function's alignment, so we
-      // can't tell whether or not it will insert nops. Assume that it will.
+      // The alignment of this MBB is larger than the function's alignment, so
+      // we can't tell whether or not it will insert nops. Assume that it will.
       return alignTo(PO, Alignment) + Alignment.value() - ParentAlign.value();
     }
   };
@@ -103,7 +103,10 @@ class BranchRelaxation : public MachineFunctionPass {
   MachineBasicBlock *splitBlockBeforeInstr(MachineInstr &MI,
                                            MachineBasicBlock *DestBB);
   void adjustBlockOffsets(MachineBasicBlock &Start);
-  bool isBlockInRange(const MachineInstr &MI, const MachineBasicBlock &BB) const;
+  void adjustBlockOffsets(MachineBasicBlock &Start,
+                          MachineFunction::iterator End);
+  bool isBlockInRange(const MachineInstr &MI,
+                      const MachineBasicBlock &BB) const;
 
   bool fixupConditionalBranch(MachineInstr &MI);
   bool fixupUnconditionalBranch(MachineInstr &MI);
@@ -199,7 +202,8 @@ void BranchRelaxation::scanFunction() {
 }
 
 /// computeBlockSize - Compute the size for MBB.
-uint64_t BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) const {
+uint64_t
+BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) const {
   uint64_t Size = 0;
   for (const MachineInstr &MI : MBB)
     Size += TII->getInstSizeInBytes(MI);
@@ -227,9 +231,14 @@ unsigned BranchRelaxation::getInstrOffset(const MachineInstr &MI) const {
 }
 
 void BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start) {
+  adjustBlockOffsets(Start, MF->end());
+}
+
+void BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start,
+                                          MachineFunction::iterator End) {
   unsigned PrevNum = Start.getNumber();
   for (auto &MBB :
-       make_range(std::next(MachineFunction::iterator(Start)), MF->end())) {
+       make_range(std::next(MachineFunction::iterator(Start)), End)) {
     unsigned Num = MBB.getNumber();
     // Get the offset and known bits at the end of the layout predecessor.
     // Include the alignment of the current block.
@@ -314,8 +323,8 @@ BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI,
   // block, it may contain a tablejump.
   BlockInfo[NewBB->getNumber()].Size = computeBlockSize(*NewBB);
 
-  // All BBOffsets following these blocks must be modified.
-  adjustBlockOffsets(*OrigBB);
+  // Update the offset of the new block.
+  adjustBlockOffsets(*OrigBB, std::next(NewBB->getIterator()));
 
   // Need to fix live-in lists if we track liveness.
   if (TRI->trackLivenessAfterRegAlloc(*MF))
@@ -328,8 +337,8 @@ BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI,
 
 /// isBlockInRange - Returns true if the distance between specific MI and
 /// specific BB can fit in MI's displacement field.
-bool BranchRelaxation::isBlockInRange(
-  const MachineInstr &MI, const MachineBasicBlock &DestBB) const {
+bool BranchRelaxation::isBlockInRange(const MachineInstr &MI,
+                                      const MachineBasicBlock &DestBB) const {
   int64_t BrOffset = getInstrOffset(MI);
   int64_t DestOffset = BlockInfo[DestBB.getNumber()].Offset;
 
@@ -369,7 +378,7 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
   };
   auto insertBranch = [&](MachineBasicBlock *MBB, MachineBasicBlock *TBB,
                           MachineBasicBlock *FBB,
-                          SmallVectorImpl<MachineOperand>& Cond) {
+                          SmallVectorImpl<MachineOperand> &Cond) {
     unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
     int NewBrSize = 0;
     TII->insertBranch(*MBB, TBB, FBB, Cond, DL, &NewBrSize);
@@ -382,13 +391,18 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
     BBSize -= RemovedSize;
   };
 
-  auto finalizeBlockChanges = [&](MachineBasicBlock *MBB,
-                                  MachineBasicBlock *NewBB) {
-    // Keep the block offsets up to date.
-    adjustBlockOffsets(*MBB);
+  // Populate the block offset and live-ins for a new basic block.
+  auto updateOffsetAndLiveness = [&](MachineBasicBlock *NewBB) {
+    assert(NewBB != nullptr && "can't populate offset for nullptr");
+
+    // Keep the block offsets approximately up to date. While they will be
+    // slight underestimates, we will update them appropriately in the next
+    // scan through the function.
+    adjustBlockOffsets(*std::prev(NewBB->getIterator()),
+                       std::next(NewBB->getIterator()));
 
     // Need to fix live-in lists if we track liveness.
-    if (NewBB && TRI->trackLivenessAfterRegAlloc(*MF))
+    if (TRI->trackLivenessAfterRegAlloc(*MF))
       computeAndAddLiveIns(LiveRegs, *NewBB);
   };
 
@@ -428,7 +442,7 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
       insertBranch(MBB, NewBB, FBB, Cond);
 
       TrampolineInsertionPoint = NewBB;
-      finalizeBlockChanges(MBB, NewBB);
+      updateOffsetAndLiveness(NewBB);
       return true;
     }
 
@@ -438,6 +452,7 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
                << ".\n");
     TrampolineInsertionPoint->setIsEndSection(NewBB->isEndSection());
     MF->erase(NewBB);
+    NewBB = nullptr;
   }
 
   // Add an unconditional branch to the destination and invert the branch
@@ -464,7 +479,6 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
 
       removeBranch(MBB);
       insertBranch(MBB, FBB, TBB, Cond);
-      finalizeBlockChanges(MBB, nullptr);
       return true;
     }
     if (FBB) {
@@ -477,10 +491,11 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
       // Do it here since if there's no split, no update is needed.
       MBB->replaceSuccessor(FBB, NewBB);
       NewBB->addSuccessor(FBB);
+      updateOffsetAndLiveness(NewBB);
     }
 
-    // We now have an appropriate fall-through block in place (either naturally or
-    // just created), so we can use the inverted the condition.
+    // We now have an appropriate fall-through block in place (either naturally
+    // or just created), so we can use the inverted the condition.
     MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
 
     LLVM_DEBUG(dbgs() << "  Insert B to " << printMBBReference(*TBB)
@@ -490,8 +505,6 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
     removeBranch(MBB);
     // Insert a new conditional branch and a new unconditional branch.
     insertBranch(MBB, &NextBB, TBB, Cond);
-
-    finalizeBlockChanges(MBB, NewBB);
     return true;
   }
   // Branch cond can't be inverted.
@@ -531,7 +544,7 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
   removeBranch(MBB);
   insertBranch(MBB, NewBB, FBB, Cond);
 
-  finalizeBlockChanges(MBB, NewBB);
+  updateOffsetAndLiveness(NewBB);
   return true;
 }
 
@@ -577,8 +590,8 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
   // Create the optional restore block and, initially, place it at the end of
   // function. That block will be placed later if it's used; otherwise, it will
   // be erased.
-  MachineBasicBlock *RestoreBB = createNewBlockAfter(MF->back(),
-                                                     DestBB->getBasicBlock());
+  MachineBasicBlock *RestoreBB =
+      createNewBlockAfter(MF->back(), DestBB->getBasicBlock());
   std::prev(RestoreBB->getIterator())
       ->setIsEndSection(RestoreBB->isEndSection());
   RestoreBB->setIsEndSection(false);
@@ -589,8 +602,10 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
                                 : DestOffset - SrcOffset,
                             RS.get());
 
+  // Update the block size and offset for the BranchBB (which may be newly
+  // created).
   BlockInfo[BranchBB->getNumber()].Size = computeBlockSize(*BranchBB);
-  adjustBlockOffsets(*MBB);
+  adjustBlockOffsets(*MBB, std::next(BranchBB->getIterator()));
 
   // If RestoreBB is required, place it appropriately.
   if (!RestoreBB->empty()) {
@@ -601,6 +616,8 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
       MachineBasicBlock *NewBB = createNewBlockAfter(*TrampolineInsertionPoint);
       TII->insertUnconditionalBranch(*NewBB, DestBB, DebugLoc());
       BlockInfo[NewBB->getNumber()].Size = computeBlockSize(*NewBB);
+      adjustBlockOffsets(*TrampolineInsertionPoint,
+                         std::next(NewBB->getIterator()));
 
       // New trampolines should be inserted after NewBB.
       TrampolineInsertionPoint = NewBB;
@@ -636,8 +653,8 @@ bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
       computeAndAddLiveIns(LiveRegs, *RestoreBB);
     // Compute the restore block size.
     BlockInfo[RestoreBB->getNumber()].Size = computeBlockSize(*RestoreBB);
-    // Update the offset starting from the previous block.
-    adjustBlockOffsets(*PrevBB);
+    // Update the estimated offset for the restore block.
+    adjustBlockOffsets(*PrevBB, DestBB->getIterator());
 
     // Fix up section information for RestoreBB and DestBB
     RestoreBB->setSectionID(DestBB->getSectionID());
@@ -718,6 +735,12 @@ bool BranchRelaxation::relaxBranchInstructions() {
     }
   }
 
+  // If we relaxed a branch, we must recompute offsets for *all* basic blocks.
+  // Otherwise, we may underestimate branch distances and fail to relax a branch
+  // that has been pushed out of range.
+  if (Changed)
+    adjustBlockOffsets(MF->front());
+
   return Changed;
 }
 
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index 9d8c9119f7719..88ed2291313c9 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -222,7 +222,7 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
 
   bool IsExiting = false;
   std::set<CopyHint> CopyHints;
-  DenseMap<unsigned, float> Hint;
+  SmallDenseMap<unsigned, float, 8> Hint;
   for (MachineRegisterInfo::reg_instr_nodbg_iterator
            I = MRI.reg_instr_nodbg_begin(LI.reg()),
            E = MRI.reg_instr_nodbg_end();
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 9e42deb94903d..d180cfcea658c 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -525,8 +525,6 @@ llvm::BasicBlockSection
 codegen::getBBSectionsMode(llvm::TargetOptions &Options) {
   if (getBBSections() == "all")
     return BasicBlockSection::All;
-  else if (getBBSections() == "labels")
-    return BasicBlockSection::Labels;
   else if (getBBSections() == "none")
     return BasicBlockSection::None;
   else {
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 513914d3218fb..08ee6169ecee8 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -1034,7 +1034,7 @@ void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,
   unsigned NumValues = SplitVTs.size();
   Align BaseAlign = DL.getPrefTypeAlign(RetTy);
   unsigned AS = DL.getAllocaAddrSpace();
-  LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getPointerTo(AS)), DL);
+  LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getContext(), AS), DL);
 
   MachinePointerInfo PtrInfo(AS);
 
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 8e860a1f74029..7ff8d2446eec5 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1885,6 +1885,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_FASIN;
     case Intrinsic::atan:
       return TargetOpcode::G_FATAN;
+    case Intrinsic::atan2:
+      return TargetOpcode::G_FATAN2;
     case Intrinsic::bswap:
       return TargetOpcode::G_BSWAP;
     case Intrinsic::bitreverse:
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index c3b6b3033cf5c..2fb2d104f1ce3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4051,12 +4051,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
     auto [Res, SubByReg] = MI.getFirst2Regs();
     LLT Ty = MRI.getType(Res);
 
-    // TODO: Handle vector types once we are able to
-    // represent them.
-    if (Ty.isVector())
-      return UnableToLegalize;
-    auto SignMask =
-        MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits()));
+    auto SignMask = MIRBuilder.buildConstant(
+        Ty, APInt::getSignMask(Ty.getScalarSizeInBits()));
     MIRBuilder.buildXor(Res, SubByReg, SignMask);
     MI.eraseFromParent();
     return Legalized;
diff --git a/llvm/lib/CodeGen/GlobalMerge.cpp b/llvm/lib/CodeGen/GlobalMerge.cpp
index c31ba6b31ad9a..007bea9a6585e 100644
--- a/llvm/lib/CodeGen/GlobalMerge.cpp
+++ b/llvm/lib/CodeGen/GlobalMerge.cpp
@@ -578,12 +578,18 @@ bool GlobalMergeImpl::doMerge(const SmallVectorImpl<GlobalVariable *> &Globals,
       Globals[k]->replaceAllUsesWith(GEP);
       Globals[k]->eraseFromParent();
 
-      // When the linkage is not internal we must emit an alias for the original
-      // variable name as it may be accessed from another object. On non-Mach-O
-      // we can also emit an alias for internal linkage as it's safe to do so.
-      // It's not safe on Mach-O as the alias (and thus the portion of the
-      // MergedGlobals variable) may be dead stripped at link time.
-      if (Linkage != GlobalValue::InternalLinkage || !IsMachO) {
+      // Emit an alias for the original variable name. This is necessary for an
+      // external symbol, as it may be accessed from another object. For
+      // internal symbols, it's not strictly required, but it's useful.
+      //
+      // This _should_ also work on Mach-O ever since '.alt_entry' support was
+      // added in 2016. Unfortunately, there's a bug in ld-prime (present at
+      // least from Xcode 15.0 through Xcode 16.0), in which -dead_strip doesn't
+      // always honor alt_entry. To workaround this issue, we don't emit aliases
+      // on Mach-O. Except, we _must_ do so for external symbols. That means
+      // MergeExternal is broken with that linker. (That option is currently off
+      // by default on MachO).
+      if (!IsMachO || Linkage == GlobalValue::ExternalLinkage) {
         GlobalAlias *GA = GlobalAlias::create(Tys[StructIdxs[idx]], AddrSpace,
                                               Linkage, Name, GEP, &M);
         GA->setVisibility(Visibility);
@@ -640,6 +646,18 @@ void GlobalMergeImpl::setMustKeepGlobalVariables(Module &M) {
   }
 }
 
+// This function returns true if the given data Section name has custom
+// subsection-splitting semantics in Mach-O (such as splitting by a fixed size)
+//
+// See also ObjFile::parseSections and getRecordSize in lld/MachO/InputFiles.cpp
+static bool isSpecialMachOSection(StringRef Section) {
+  // Uses starts_with, since section attributes can appear at the end of the
+  // name.
+  return Section.starts_with("__DATA,__cfstring") ||
+         Section.starts_with("__DATA,__objc_classrefs") ||
+         Section.starts_with("__DATA,__objc_selrefs");
+}
+
 bool GlobalMergeImpl::run(Module &M) {
   if (!EnableGlobalMerge)
     return false;
@@ -678,6 +696,10 @@ bool GlobalMergeImpl::run(Module &M) {
     unsigned AddressSpace = PT->getAddressSpace();
     StringRef Section = GV.getSection();
 
+    // On Mach-O, some section names have special semantics. Don't merge these.
+    if (IsMachO && isSpecialMachOSection(Section))
+      continue;
+
     // Ignore all 'special' globals.
     if (GV.getName().starts_with("llvm.") || GV.getName().starts_with(".llvm."))
       continue;
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index 27f0a9331a3e3..a0f0e27478d02 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -780,7 +780,7 @@ bool MIParser::parseBasicBlockDefinition(
                             "' is not defined in the function '" +
                             MF.getName() + "'");
   }
-  auto *MBB = MF.CreateMachineBasicBlock(BB);
+  auto *MBB = MF.CreateMachineBasicBlock(BB, BBID);
   MF.insert(MF.end(), MBB);
   bool WasInserted = MBBSlots.insert(std::make_pair(ID, MBB)).second;
   if (!WasInserted)
@@ -799,13 +799,6 @@ bool MIParser::parseBasicBlockDefinition(
     MBB->setSectionID(*SectionID);
     MF.setBBSectionsType(BasicBlockSection::List);
   }
-  if (BBID.has_value()) {
-    // BBSectionsType is set to `List` if any basic blocks has `SectionID`.
-    // Here, we set it to `Labels` if it hasn't been set above.
-    if (!MF.hasBBSections())
-      MF.setBBSectionsType(BasicBlockSection::Labels);
-    MBB->setBBID(BBID.value());
-  }
   MBB->setCallFrameSize(CallFrameSize);
   return false;
 }
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index be07fbf478b1d..997c428ca77dc 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -569,9 +569,7 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF,
     return true;
   }
   // Check Basic Block Section Flags.
-  if (MF.getTarget().getBBSectionsType() == BasicBlockSection::Labels) {
-    MF.setBBSectionsType(BasicBlockSection::Labels);
-  } else if (MF.hasBBSections()) {
+  if (MF.hasBBSections()) {
     MF.assignBeginEndSections();
   }
   PFS.SM = &SM;
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
index ccfc4565d3a9b..49c8a0e466337 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.cpp
@@ -39,8 +39,6 @@ VRegRenamer::getVRegRenameMap(const std::vector<NamedVReg> &VRegs) {
   StringMap<unsigned> VRegNameCollisionMap;
 
   auto GetUniqueVRegName = [&VRegNameCollisionMap](const NamedVReg &Reg) {
-    if (!VRegNameCollisionMap.contains(Reg.getName()))
-      VRegNameCollisionMap[Reg.getName()] = 0;
     const unsigned Counter = ++VRegNameCollisionMap[Reg.getName()];
     return Reg.getName() + "__" + std::to_string(Counter);
   };
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index a52c82d77ca64..7807875c06584 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -3619,9 +3619,8 @@ void MachineBlockPlacement::applyExtTsp() {
                     << " with profile = " << F->getFunction().hasProfileData()
                     << " (" << F->getName().str() << ")"
                     << "\n");
-  LLVM_DEBUG(
-      dbgs() << format("  original  layout score: %0.2f\n",
-                       calcExtTspScore(BlockSizes, BlockCounts, JumpCounts)));
+  LLVM_DEBUG(dbgs() << format("  original  layout score: %0.2f\n",
+                              calcExtTspScore(BlockSizes, JumpCounts)));
 
   // Run the layout algorithm.
   auto NewOrder = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
@@ -3630,9 +3629,9 @@ void MachineBlockPlacement::applyExtTsp() {
   for (uint64_t Node : NewOrder) {
     NewBlockOrder.push_back(CurrentBlockOrder[Node]);
   }
-  LLVM_DEBUG(dbgs() << format("  optimized layout score: %0.2f\n",
-                              calcExtTspScore(NewOrder, BlockSizes, BlockCounts,
-                                              JumpCounts)));
+  LLVM_DEBUG(
+      dbgs() << format("  optimized layout score: %0.2f\n",
+                       calcExtTspScore(NewOrder, BlockSizes, JumpCounts)));
 
   // Assign new block order.
   assignBlockOrder(NewBlockOrder);
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index ab45663436ced..b56888a0f71fe 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -465,11 +465,9 @@ MachineFunction::CreateMachineBasicBlock(const BasicBlock *BB,
   MachineBasicBlock *MBB =
       new (BasicBlockRecycler.Allocate<MachineBasicBlock>(Allocator))
           MachineBasicBlock(*this, BB);
-  // Set BBID for `-basic-block=sections=labels` and
-  // `-basic-block-sections=list` to allow robust mapping of profiles to basic
-  // blocks.
-  if (Target.getBBSectionsType() == BasicBlockSection::Labels ||
-      Target.Options.BBAddrMap ||
+  // Set BBID for `-basic-block-sections=list` and `-basic-block-address-map` to
+  // allow robust mapping of profiles to basic blocks.
+  if (Target.Options.BBAddrMap ||
       Target.getBBSectionsType() == BasicBlockSection::List)
     MBB->setBBID(BBID.has_value() ? *BBID : UniqueBBID{NextBBID++, 0});
   return MBB;
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 6768eeeb4364c..3289a692221ba 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -239,7 +239,7 @@ namespace {
 
     bool IsCheapInstruction(MachineInstr &MI) const;
 
-    bool CanCauseHighRegPressure(const DenseMap<unsigned, int> &Cost,
+    bool CanCauseHighRegPressure(const SmallDenseMap<unsigned, int> &Cost,
                                  bool Cheap);
 
     void UpdateBackTraceRegPressure(const MachineInstr *MI);
@@ -264,9 +264,9 @@ namespace {
 
     void InitRegPressure(MachineBasicBlock *BB);
 
-    DenseMap<unsigned, int> calcRegisterCost(const MachineInstr *MI,
-                                             bool ConsiderSeen,
-                                             bool ConsiderUnseenAsDef);
+    SmallDenseMap<unsigned, int> calcRegisterCost(const MachineInstr *MI,
+                                                  bool ConsiderSeen,
+                                                  bool ConsiderUnseenAsDef);
 
     void UpdateRegPressure(const MachineInstr *MI,
                            bool ConsiderUnseenAsDef = false);
@@ -977,10 +977,10 @@ void MachineLICMImpl::UpdateRegPressure(const MachineInstr *MI,
 /// If 'ConsiderSeen' is true, updates 'RegSeen' and uses the information to
 /// figure out which usages are live-ins.
 /// FIXME: Figure out a way to consider 'RegSeen' from all code paths.
-DenseMap<unsigned, int>
+SmallDenseMap<unsigned, int>
 MachineLICMImpl::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
                                   bool ConsiderUnseenAsDef) {
-  DenseMap<unsigned, int> Cost;
+  SmallDenseMap<unsigned, int> Cost;
   if (MI->isImplicitDef())
     return Cost;
   for (unsigned i = 0, e = MI->getDesc().getNumOperands(); i != e; ++i) {
@@ -1248,7 +1248,7 @@ bool MachineLICMImpl::IsCheapInstruction(MachineInstr &MI) const {
 /// Visit BBs from header to current BB, check if hoisting an instruction of the
 /// given cost matrix can cause high register pressure.
 bool MachineLICMImpl::CanCauseHighRegPressure(
-    const DenseMap<unsigned, int> &Cost, bool CheapInstr) {
+    const SmallDenseMap<unsigned, int> &Cost, bool CheapInstr) {
   for (const auto &RPIdAndCost : Cost) {
     if (RPIdAndCost.second <= 0)
       continue;
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 53ce21906204c..12a48ab06f1c0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -82,8 +82,7 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
 /// implicit physical register output.
 void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
-                                   Register SrcReg,
-                                   DenseMap<SDValue, Register> &VRBaseMap) {
+                                   Register SrcReg, VRBaseMapType &VRBaseMap) {
   Register VRBase;
   if (SrcReg.isVirtual()) {
     // Just use the input register directly!
@@ -187,7 +186,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
                                        MachineInstrBuilder &MIB,
                                        const MCInstrDesc &II,
                                        bool IsClone, bool IsCloned,
-                                       DenseMap<SDValue, Register> &VRBaseMap) {
+                                       VRBaseMapType &VRBaseMap) {
   assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
          "IMPLICIT_DEF should have been handled as a special case elsewhere!");
 
@@ -265,8 +264,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
 
 /// getVR - Return the virtual register corresponding to the specified result
 /// of the specified node.
-Register InstrEmitter::getVR(SDValue Op,
-                             DenseMap<SDValue, Register> &VRBaseMap) {
+Register InstrEmitter::getVR(SDValue Op, VRBaseMapType &VRBaseMap) {
   if (Op.isMachineOpcode() &&
       Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
     // Add an IMPLICIT_DEF instruction before every use.
@@ -280,7 +278,7 @@ Register InstrEmitter::getVR(SDValue Op,
     return VReg;
   }
 
-  DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
+  VRBaseMapType::iterator I = VRBaseMap.find(Op);
   assert(I != VRBaseMap.end() && "Node emitted out of order - late");
   return I->second;
 }
@@ -318,7 +316,7 @@ InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
                                  SDValue Op,
                                  unsigned IIOpNum,
                                  const MCInstrDesc *II,
-                                 DenseMap<SDValue, Register> &VRBaseMap,
+                                 VRBaseMapType &VRBaseMap,
                                  bool IsDebug, bool IsClone, bool IsCloned) {
   assert(Op.getValueType() != MVT::Other &&
          Op.getValueType() != MVT::Glue &&
@@ -395,12 +393,10 @@ InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
 /// AddOperand - Add the specified operand to the specified machine instr.  II
 /// specifies the instruction information for the node, and IIOpNum is the
 /// operand number (in the II) that we are adding.
-void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
-                              SDValue Op,
-                              unsigned IIOpNum,
-                              const MCInstrDesc *II,
-                              DenseMap<SDValue, Register> &VRBaseMap,
-                              bool IsDebug, bool IsClone, bool IsCloned) {
+void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, SDValue Op,
+                              unsigned IIOpNum, const MCInstrDesc *II,
+                              VRBaseMapType &VRBaseMap, bool IsDebug,
+                              bool IsClone, bool IsCloned) {
   if (Op.isMachineOpcode()) {
     AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
                        IsDebug, IsClone, IsCloned);
@@ -499,8 +495,7 @@ Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
 
 /// EmitSubregNode - Generate machine code for subreg nodes.
 ///
-void InstrEmitter::EmitSubregNode(SDNode *Node,
-                                  DenseMap<SDValue, Register> &VRBaseMap,
+void InstrEmitter::EmitSubregNode(SDNode *Node, VRBaseMapType &VRBaseMap,
                                   bool IsClone, bool IsCloned) {
   Register VRBase;
   unsigned Opc = Node->getMachineOpcode();
@@ -634,7 +629,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node,
 ///
 void
 InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
-                                     DenseMap<SDValue, Register> &VRBaseMap) {
+                                     VRBaseMapType &VRBaseMap) {
   Register VReg = getVR(Node->getOperand(0), VRBaseMap);
 
   // Create the new VReg in the destination class and emit a copy.
@@ -653,9 +648,8 @@ InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
 
 /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
 ///
-void InstrEmitter::EmitRegSequence(SDNode *Node,
-                                  DenseMap<SDValue, Register> &VRBaseMap,
-                                  bool IsClone, bool IsCloned) {
+void InstrEmitter::EmitRegSequence(SDNode *Node, VRBaseMapType &VRBaseMap,
+                                   bool IsClone, bool IsCloned) {
   unsigned DstRCIdx = Node->getConstantOperandVal(0);
   const TargetRegisterClass *RC = TRI->getRegClass(DstRCIdx);
   Register NewVReg = MRI->createVirtualRegister(TRI->getAllocatableClass(RC));
@@ -703,7 +697,7 @@ void InstrEmitter::EmitRegSequence(SDNode *Node,
 ///
 MachineInstr *
 InstrEmitter::EmitDbgValue(SDDbgValue *SD,
-                           DenseMap<SDValue, Register> &VRBaseMap) {
+                           VRBaseMapType &VRBaseMap) {
   DebugLoc DL = SD->getDebugLoc();
   assert(cast<DILocalVariable>(SD->getVariable())
              ->isValidLocationForIntrinsic(DL) &&
@@ -755,7 +749,7 @@ MachineOperand GetMOForConstDbgOp(const SDDbgOperand &Op) {
 void InstrEmitter::AddDbgValueLocationOps(
     MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
     ArrayRef<SDDbgOperand> LocationOps,
-    DenseMap<SDValue, Register> &VRBaseMap) {
+    VRBaseMapType &VRBaseMap) {
   for (const SDDbgOperand &Op : LocationOps) {
     switch (Op.getKind()) {
     case SDDbgOperand::FRAMEIX:
@@ -786,7 +780,7 @@ void InstrEmitter::AddDbgValueLocationOps(
 
 MachineInstr *
 InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
-                              DenseMap<SDValue, Register> &VRBaseMap) {
+                              VRBaseMapType &VRBaseMap) {
   MDNode *Var = SD->getVariable();
   const DIExpression *Expr = (DIExpression *)SD->getExpression();
   DebugLoc DL = SD->getDebugLoc();
@@ -862,7 +856,7 @@ InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
       // Look up the corresponding VReg for the given SDNode, if any.
       SDNode *Node = DbgOperand.getSDNode();
       SDValue Op = SDValue(Node, DbgOperand.getResNo());
-      DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Op);
+      VRBaseMapType::iterator I = VRBaseMap.find(Op);
       // No VReg -> produce a DBG_VALUE $noreg instead.
       if (I == VRBaseMap.end())
         break;
@@ -928,7 +922,7 @@ MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
 
 MachineInstr *
 InstrEmitter::EmitDbgValueList(SDDbgValue *SD,
-                               DenseMap<SDValue, Register> &VRBaseMap) {
+                               VRBaseMapType &VRBaseMap) {
   MDNode *Var = SD->getVariable();
   DIExpression *Expr = SD->getExpression();
   DebugLoc DL = SD->getDebugLoc();
@@ -944,7 +938,7 @@ InstrEmitter::EmitDbgValueList(SDDbgValue *SD,
 
 MachineInstr *
 InstrEmitter::EmitDbgValueFromSingleOp(SDDbgValue *SD,
-                                       DenseMap<SDValue, Register> &VRBaseMap) {
+                                       VRBaseMapType &VRBaseMap) {
   MDNode *Var = SD->getVariable();
   DIExpression *Expr = SD->getExpression();
   DebugLoc DL = SD->getDebugLoc();
@@ -996,7 +990,7 @@ InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
 ///
 void InstrEmitter::
 EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
-                DenseMap<SDValue, Register> &VRBaseMap) {
+                VRBaseMapType &VRBaseMap) {
   unsigned Opc = Node->getMachineOpcode();
 
   // Handle subreg insert/extract specially
@@ -1238,7 +1232,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
 /// needed dependencies.
 void InstrEmitter::
 EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
-                DenseMap<SDValue, Register> &VRBaseMap) {
+                VRBaseMapType &VRBaseMap) {
   switch (Node->getOpcode()) {
   default:
 #ifndef NDEBUG
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 959bce31c8b27..16d754cdc2338 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -30,6 +30,10 @@ class TargetLowering;
 class TargetMachine;
 
 class LLVM_LIBRARY_VISIBILITY InstrEmitter {
+public:
+  using VRBaseMapType = SmallDenseMap<SDValue, Register, 16>;
+
+private:
   MachineFunction *MF;
   MachineRegisterInfo *MRI;
   const TargetInstrInfo *TII;
@@ -45,18 +49,17 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
   /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
   /// implicit physical register output.
   void EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
-                       Register SrcReg, DenseMap<SDValue, Register> &VRBaseMap);
+                       Register SrcReg, VRBaseMapType &VRBaseMap);
 
   void CreateVirtualRegisters(SDNode *Node,
                               MachineInstrBuilder &MIB,
                               const MCInstrDesc &II,
                               bool IsClone, bool IsCloned,
-                              DenseMap<SDValue, Register> &VRBaseMap);
+                              VRBaseMapType &VRBaseMap);
 
   /// getVR - Return the virtual register corresponding to the specified result
   /// of the specified node.
-  Register getVR(SDValue Op,
-                 DenseMap<SDValue, Register> &VRBaseMap);
+  Register getVR(SDValue Op, VRBaseMapType &VRBaseMap);
 
   /// AddRegisterOperand - Add the specified register as an operand to the
   /// specified machine instr. Insert register copies if the register is
@@ -65,7 +68,7 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
                           SDValue Op,
                           unsigned IIOpNum,
                           const MCInstrDesc *II,
-                          DenseMap<SDValue, Register> &VRBaseMap,
+                          VRBaseMapType &VRBaseMap,
                           bool IsDebug, bool IsClone, bool IsCloned);
 
   /// AddOperand - Add the specified operand to the specified machine instr.  II
@@ -76,7 +79,7 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
                   SDValue Op,
                   unsigned IIOpNum,
                   const MCInstrDesc *II,
-                  DenseMap<SDValue, Register> &VRBaseMap,
+                  VRBaseMapType &VRBaseMap,
                   bool IsDebug, bool IsClone, bool IsCloned);
 
   /// ConstrainForSubReg - Try to constrain VReg to a register class that
@@ -87,20 +90,20 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
 
   /// EmitSubregNode - Generate machine code for subreg nodes.
   ///
-  void EmitSubregNode(SDNode *Node, DenseMap<SDValue, Register> &VRBaseMap,
-                      bool IsClone, bool IsCloned);
+  void EmitSubregNode(SDNode *Node, VRBaseMapType &VRBaseMap, bool IsClone,
+                      bool IsCloned);
 
   /// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
   /// COPY_TO_REGCLASS is just a normal copy, except that the destination
   /// register is constrained to be in a particular register class.
   ///
-  void EmitCopyToRegClassNode(SDNode *Node,
-                              DenseMap<SDValue, Register> &VRBaseMap);
+  void EmitCopyToRegClassNode(SDNode *Node, VRBaseMapType &VRBaseMap);
 
   /// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
   ///
-  void EmitRegSequence(SDNode *Node, DenseMap<SDValue, Register> &VRBaseMap,
-                       bool IsClone, bool IsCloned);
+  void EmitRegSequence(SDNode *Node, VRBaseMapType &VRBaseMap, bool IsClone,
+                       bool IsCloned);
+
 public:
   /// CountResults - The results of target nodes have register or immediate
   /// operands first, then an optional chain, and optional flag operands
@@ -110,29 +113,26 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
   void AddDbgValueLocationOps(MachineInstrBuilder &MIB,
                               const MCInstrDesc &DbgValDesc,
                               ArrayRef<SDDbgOperand> Locations,
-                              DenseMap<SDValue, Register> &VRBaseMap);
+                              VRBaseMapType &VRBaseMap);
 
   /// EmitDbgValue - Generate machine instruction for a dbg_value node.
   ///
-  MachineInstr *EmitDbgValue(SDDbgValue *SD,
-                             DenseMap<SDValue, Register> &VRBaseMap);
+  MachineInstr *EmitDbgValue(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
 
   /// Emit a dbg_value as a DBG_INSTR_REF. May produce DBG_VALUE $noreg instead
   /// if there is no variable location; alternately a half-formed DBG_INSTR_REF
   /// that refers to a virtual register and is corrected later in isel.
-  MachineInstr *EmitDbgInstrRef(SDDbgValue *SD,
-                                DenseMap<SDValue, Register> &VRBaseMap);
+  MachineInstr *EmitDbgInstrRef(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
 
   /// Emit a DBG_VALUE $noreg, indicating a variable has no location.
   MachineInstr *EmitDbgNoLocation(SDDbgValue *SD);
 
   /// Emit a DBG_VALUE_LIST from the operands to SDDbgValue.
-  MachineInstr *EmitDbgValueList(SDDbgValue *SD,
-                                 DenseMap<SDValue, Register> &VRBaseMap);
+  MachineInstr *EmitDbgValueList(SDDbgValue *SD, VRBaseMapType &VRBaseMap);
 
   /// Emit a DBG_VALUE from the operands to SDDbgValue.
   MachineInstr *EmitDbgValueFromSingleOp(SDDbgValue *SD,
-                                    DenseMap<SDValue, Register> &VRBaseMap);
+                                         VRBaseMapType &VRBaseMap);
 
   /// Generate machine instruction for a dbg_label node.
   MachineInstr *EmitDbgLabel(SDDbgLabel *SD);
@@ -140,7 +140,7 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
   /// EmitNode - Generate machine code for a node and needed dependencies.
   ///
   void EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
-                DenseMap<SDValue, Register> &VRBaseMap) {
+                VRBaseMapType &VRBaseMap) {
     if (Node->isMachineOpcode())
       EmitMachineNode(Node, IsClone, IsCloned, VRBaseMap);
     else
@@ -160,9 +160,9 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
 
 private:
   void EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
-                       DenseMap<SDValue, Register> &VRBaseMap);
+                       VRBaseMapType &VRBaseMap);
   void EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
-                       DenseMap<SDValue, Register> &VRBaseMap);
+                       VRBaseMapType &VRBaseMap);
 };
 } // namespace llvm
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index de4a1ac2a3baf..70a7438440191 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -770,7 +770,7 @@ void ScheduleDAGLinearize::Schedule() {
 MachineBasicBlock*
 ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
   InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos);
-  DenseMap<SDValue, Register> VRBaseMap;
+  InstrEmitter::VRBaseMapType VRBaseMap;
 
   LLVM_DEBUG({ dbgs() << "\n*** Final schedule ***\n"; });
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 53dd71d173473..31939ae5922ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -737,7 +737,7 @@ void ScheduleDAGSDNodes::VerifyScheduledSequence(bool isBottomUp) {
 static void
 ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
                    SmallVectorImpl<std::pair<unsigned, MachineInstr*> > &Orders,
-                   DenseMap<SDValue, Register> &VRBaseMap, unsigned Order) {
+                   InstrEmitter::VRBaseMapType &VRBaseMap, unsigned Order) {
   if (!N->getHasDebugValue())
     return;
 
@@ -782,7 +782,7 @@ ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
 // instructions in the right order.
 static void
 ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
-                  DenseMap<SDValue, Register> &VRBaseMap,
+                  InstrEmitter::VRBaseMapType &VRBaseMap,
                   SmallVectorImpl<std::pair<unsigned, MachineInstr *>> &Orders,
                   SmallSet<Register, 8> &Seen, MachineInstr *NewInsn) {
   unsigned Order = N->getIROrder();
@@ -808,7 +808,7 @@ ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
 }
 
 void ScheduleDAGSDNodes::
-EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, Register> &VRBaseMap,
+EmitPhysRegCopy(SUnit *SU, SmallDenseMap<SUnit *, Register, 16> &VRBaseMap,
                 MachineBasicBlock::iterator InsertPos) {
   for (const SDep &Pred : SU->Preds) {
     if (Pred.isCtrl())
@@ -851,8 +851,8 @@ EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, Register> &VRBaseMap,
 MachineBasicBlock *ScheduleDAGSDNodes::
 EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
   InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos);
-  DenseMap<SDValue, Register> VRBaseMap;
-  DenseMap<SUnit*, Register> CopyVRBaseMap;
+  InstrEmitter::VRBaseMapType VRBaseMap;
+  SmallDenseMap<SUnit *, Register, 16> CopyVRBaseMap;
   SmallVector<std::pair<unsigned, MachineInstr*>, 32> Orders;
   SmallSet<Register, 8> Seen;
   bool HasDbg = DAG->hasDebugValues();
@@ -861,7 +861,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
   // Zero, one, or multiple instructions can be created when emitting a node.
   auto EmitNode =
       [&](SDNode *Node, bool IsClone, bool IsCloned,
-          DenseMap<SDValue, Register> &VRBaseMap) -> MachineInstr * {
+          InstrEmitter::VRBaseMapType &VRBaseMap) -> MachineInstr * {
     // Fetch instruction prior to this, or end() if nonexistant.
     auto GetPrevInsn = [&](MachineBasicBlock::iterator I) {
       if (I == BB->begin())
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 446df640821d8..b7d25c6ccc9b0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -184,7 +184,8 @@ class InstrItineraryData;
     void BuildSchedUnits();
     void AddSchedEdges();
 
-    void EmitPhysRegCopy(SUnit *SU, DenseMap<SUnit*, Register> &VRBaseMap,
+    void EmitPhysRegCopy(SUnit *SU,
+                         SmallDenseMap<SUnit *, Register, 16> &VRBaseMap,
                          MachineBasicBlock::iterator InsertPos);
   };
 
diff --git a/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp b/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp
index 7d054cb7c7c71..4ee86e07e1a5a 100644
--- a/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp
+++ b/llvm/lib/CodeGen/TargetFrameLoweringImpl.cpp
@@ -103,15 +103,18 @@ void TargetFrameLowering::determineCalleeSaves(MachineFunction &MF,
   // saved registers.
   SavedRegs.resize(TRI.getNumRegs());
 
-  // When interprocedural register allocation is enabled caller saved registers
-  // are preferred over callee saved registers.
+  // Get the callee saved register list...
+  const MCPhysReg *CSRegs = nullptr;
+
+  // When interprocedural register allocation is enabled, callee saved register
+  // list should be empty, since caller saved registers are preferred over
+  // callee saved registers. Unless it has some risked CSR to be optimized out.
   if (MF.getTarget().Options.EnableIPRA &&
       isSafeForNoCSROpt(MF.getFunction()) &&
       isProfitableForNoCSROpt(MF.getFunction()))
-    return;
-
-  // Get the callee saved register list...
-  const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+    CSRegs = TRI.getIPRACSRegs(&MF);
+  else
+    CSRegs = MF.getRegInfo().getCalleeSavedRegs();
 
   // Early exit if there are no callee saved registers.
   if (!CSRegs || CSRegs[0] == 0)
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 02d1d9d9f7898..3390d651d6c69 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1272,6 +1272,9 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
         // nvvm.bitcast.{f2i,i2f,ll2d,d2ll}
         Expand =
             Name == "f2i" || Name == "i2f" || Name == "ll2d" || Name == "d2ll";
+      else if (Name.consume_front("rotate."))
+        // nvvm.rotate.{b32,b64,right.b64}
+        Expand = Name == "b32" || Name == "b64" || Name == "right.b64";
       else
         Expand = false;
 
@@ -2258,6 +2261,108 @@ void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
   }
 }
 
+static Value *upgradeNVVMIntrinsicCall(StringRef Name, CallBase *CI,
+                                       Function *F, IRBuilder<> &Builder) {
+  Value *Rep = nullptr;
+
+  if (Name == "abs.i" || Name == "abs.ll") {
+    Value *Arg = CI->getArgOperand(0);
+    Value *Neg = Builder.CreateNeg(Arg, "neg");
+    Value *Cmp = Builder.CreateICmpSGE(
+        Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
+    Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
+  } else if (Name.starts_with("atomic.load.add.f32.p") ||
+             Name.starts_with("atomic.load.add.f64.p")) {
+    Value *Ptr = CI->getArgOperand(0);
+    Value *Val = CI->getArgOperand(1);
+    Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
+                                  AtomicOrdering::SequentiallyConsistent);
+  } else if (Name.consume_front("max.") &&
+             (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
+              Name == "ui" || Name == "ull")) {
+    Value *Arg0 = CI->getArgOperand(0);
+    Value *Arg1 = CI->getArgOperand(1);
+    Value *Cmp = Name.starts_with("u")
+                     ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
+                     : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
+    Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
+  } else if (Name.consume_front("min.") &&
+             (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
+              Name == "ui" || Name == "ull")) {
+    Value *Arg0 = CI->getArgOperand(0);
+    Value *Arg1 = CI->getArgOperand(1);
+    Value *Cmp = Name.starts_with("u")
+                     ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
+                     : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
+    Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
+  } else if (Name == "clz.ll") {
+    // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
+    Value *Arg = CI->getArgOperand(0);
+    Value *Ctlz = Builder.CreateCall(
+        Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
+                                  {Arg->getType()}),
+        {Arg, Builder.getFalse()}, "ctlz");
+    Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
+  } else if (Name == "popc.ll") {
+    // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
+    // i64.
+    Value *Arg = CI->getArgOperand(0);
+    Value *Popc = Builder.CreateCall(
+        Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
+                                  {Arg->getType()}),
+        Arg, "ctpop");
+    Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
+  } else if (Name == "h2f") {
+    Rep = Builder.CreateCall(
+        Intrinsic::getDeclaration(F->getParent(), Intrinsic::convert_from_fp16,
+                                  {Builder.getFloatTy()}),
+        CI->getArgOperand(0), "h2f");
+  } else if (Name.consume_front("bitcast.") &&
+             (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
+              Name == "d2ll")) {
+    Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
+  } else if (Name == "rotate.b32") {
+    Value *Arg = CI->getOperand(0);
+    Value *ShiftAmt = CI->getOperand(1);
+    Rep = Builder.CreateIntrinsic(Builder.getInt32Ty(), Intrinsic::fshl,
+                                  {Arg, Arg, ShiftAmt});
+  } else if (Name == "rotate.b64") {
+    Type *Int64Ty = Builder.getInt64Ty();
+    Value *Arg = CI->getOperand(0);
+    Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
+    Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshl,
+                                  {Arg, Arg, ZExtShiftAmt});
+  } else if (Name == "rotate.right.b64") {
+    Type *Int64Ty = Builder.getInt64Ty();
+    Value *Arg = CI->getOperand(0);
+    Value *ZExtShiftAmt = Builder.CreateZExt(CI->getOperand(1), Int64Ty);
+    Rep = Builder.CreateIntrinsic(Int64Ty, Intrinsic::fshr,
+                                  {Arg, Arg, ZExtShiftAmt});
+  } else {
+    Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
+    if (IID != Intrinsic::not_intrinsic &&
+        !F->getReturnType()->getScalarType()->isBFloatTy()) {
+      rename(F);
+      Function *NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
+      SmallVector<Value *, 2> Args;
+      for (size_t I = 0; I < NewFn->arg_size(); ++I) {
+        Value *Arg = CI->getArgOperand(I);
+        Type *OldType = Arg->getType();
+        Type *NewType = NewFn->getArg(I)->getType();
+        Args.push_back(
+            (OldType->isIntegerTy() && NewType->getScalarType()->isBFloatTy())
+                ? Builder.CreateBitCast(Arg, NewType)
+                : Arg);
+      }
+      Rep = Builder.CreateCall(NewFn, Args);
+      if (F->getReturnType()->isIntegerTy())
+        Rep = Builder.CreateBitCast(Rep, F->getReturnType());
+    }
+  }
+
+  return Rep;
+}
+
 static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
                                       IRBuilder<> &Builder) {
   LLVMContext &C = F->getContext();
@@ -4208,85 +4313,8 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
 
     if (!IsX86 && Name == "stackprotectorcheck") {
       Rep = nullptr;
-    } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
-      Value *Arg = CI->getArgOperand(0);
-      Value *Neg = Builder.CreateNeg(Arg, "neg");
-      Value *Cmp = Builder.CreateICmpSGE(
-          Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
-      Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
-    } else if (IsNVVM && (Name.starts_with("atomic.load.add.f32.p") ||
-                          Name.starts_with("atomic.load.add.f64.p"))) {
-      Value *Ptr = CI->getArgOperand(0);
-      Value *Val = CI->getArgOperand(1);
-      Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, MaybeAlign(),
-                                    AtomicOrdering::SequentiallyConsistent);
-    } else if (IsNVVM && Name.consume_front("max.") &&
-               (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
-                Name == "ui" || Name == "ull")) {
-      Value *Arg0 = CI->getArgOperand(0);
-      Value *Arg1 = CI->getArgOperand(1);
-      Value *Cmp = Name.starts_with("u")
-                       ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
-                       : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
-      Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
-    } else if (IsNVVM && Name.consume_front("min.") &&
-               (Name == "s" || Name == "i" || Name == "ll" || Name == "us" ||
-                Name == "ui" || Name == "ull")) {
-      Value *Arg0 = CI->getArgOperand(0);
-      Value *Arg1 = CI->getArgOperand(1);
-      Value *Cmp = Name.starts_with("u")
-                       ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
-                       : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
-      Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
-    } else if (IsNVVM && Name == "clz.ll") {
-      // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 returns an i64.
-      Value *Arg = CI->getArgOperand(0);
-      Value *Ctlz = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
-                                    {Arg->getType()}),
-          {Arg, Builder.getFalse()}, "ctlz");
-      Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
-    } else if (IsNVVM && Name == "popc.ll") {
-      // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 returns an
-      // i64.
-      Value *Arg = CI->getArgOperand(0);
-      Value *Popc = Builder.CreateCall(
-          Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
-                                    {Arg->getType()}),
-          Arg, "ctpop");
-      Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
     } else if (IsNVVM) {
-      if (Name == "h2f") {
-        Rep =
-            Builder.CreateCall(Intrinsic::getDeclaration(
-                                   F->getParent(), Intrinsic::convert_from_fp16,
-                                   {Builder.getFloatTy()}),
-                               CI->getArgOperand(0), "h2f");
-      } else if (Name.consume_front("bitcast.") &&
-                 (Name == "f2i" || Name == "i2f" || Name == "ll2d" ||
-                  Name == "d2ll")) {
-        Rep = Builder.CreateBitCast(CI->getArgOperand(0), CI->getType());
-      } else {
-        Intrinsic::ID IID = shouldUpgradeNVPTXBF16Intrinsic(Name);
-        if (IID != Intrinsic::not_intrinsic &&
-            !F->getReturnType()->getScalarType()->isBFloatTy()) {
-          rename(F);
-          NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
-          SmallVector<Value *, 2> Args;
-          for (size_t I = 0; I < NewFn->arg_size(); ++I) {
-            Value *Arg = CI->getArgOperand(I);
-            Type *OldType = Arg->getType();
-            Type *NewType = NewFn->getArg(I)->getType();
-            Args.push_back((OldType->isIntegerTy() &&
-                            NewType->getScalarType()->isBFloatTy())
-                               ? Builder.CreateBitCast(Arg, NewType)
-                               : Arg);
-          }
-          Rep = Builder.CreateCall(NewFn, Args);
-          if (F->getReturnType()->isIntegerTy())
-            Rep = Builder.CreateBitCast(Rep, F->getReturnType());
-        }
-      }
+      Rep = upgradeNVVMIntrinsicCall(Name, CI, F, Builder);
     } else if (IsX86) {
       Rep = upgradeX86IntrinsicCall(Name, CI, F, Builder);
     } else if (IsARM) {
diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp
index a5de02abce667..178b1d21e5200 100644
--- a/llvm/lib/MC/MCRegisterInfo.cpp
+++ b/llvm/lib/MC/MCRegisterInfo.cpp
@@ -141,7 +141,7 @@ unsigned MCRegisterInfo::getSubRegIndex(MCRegister Reg,
   return 0;
 }
 
-int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const {
+int64_t MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const {
   const DwarfLLVMRegPair *M = isEH ? EHL2DwarfRegs : L2DwarfRegs;
   unsigned Size = isEH ? EHL2DwarfRegsSize : L2DwarfRegsSize;
 
@@ -151,24 +151,28 @@ int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const {
   const DwarfLLVMRegPair *I = std::lower_bound(M, M+Size, Key);
   if (I == M+Size || I->FromReg != RegNum)
     return -1;
-  return I->ToReg;
+  // Consumers need to be able to detect -1 and -2, but at various points
+  // the numbers move between unsigned and signed representations, as well as
+  // between 32- and 64-bit representations. We need to convert first to int
+  // before int64_t for proper sign handling.
+  return int64_t(int(I->ToReg));
 }
 
-std::optional<MCRegister> MCRegisterInfo::getLLVMRegNum(unsigned RegNum,
+std::optional<MCRegister> MCRegisterInfo::getLLVMRegNum(uint64_t RegNum,
                                                         bool isEH) const {
   const DwarfLLVMRegPair *M = isEH ? EHDwarf2LRegs : Dwarf2LRegs;
   unsigned Size = isEH ? EHDwarf2LRegsSize : Dwarf2LRegsSize;
 
   if (!M)
     return std::nullopt;
-  DwarfLLVMRegPair Key = { RegNum, 0 };
+  DwarfLLVMRegPair Key = {unsigned(RegNum), 0};
   const DwarfLLVMRegPair *I = std::lower_bound(M, M+Size, Key);
   if (I != M + Size && I->FromReg == RegNum)
     return MCRegister::from(I->ToReg);
   return std::nullopt;
 }
 
-int MCRegisterInfo::getDwarfRegNumFromDwarfEHRegNum(unsigned RegNum) const {
+int64_t MCRegisterInfo::getDwarfRegNumFromDwarfEHRegNum(uint64_t RegNum) const {
   // On ELF platforms, DWARF EH register numbers are the same as DWARF
   // other register numbers.  On Darwin x86, they differ and so need to be
   // mapped.  The .cfi_* directives accept integer literals as well as
diff --git a/llvm/lib/SandboxIR/Argument.cpp b/llvm/lib/SandboxIR/Argument.cpp
new file mode 100644
index 0000000000000..e35da2d1dbcb7
--- /dev/null
+++ b/llvm/lib/SandboxIR/Argument.cpp
@@ -0,0 +1,23 @@
+//===- Argument.cpp - The function Argument class of Sandbox IR -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Argument.h"
+
+namespace llvm::sandboxir {
+
+#ifndef NDEBUG
+void Argument::printAsOperand(raw_ostream &OS) const {
+  printAsOperandCommon(OS);
+}
+void Argument::dumpOS(raw_ostream &OS) const {
+  dumpCommonPrefix(OS);
+  dumpCommonSuffix(OS);
+}
+#endif // NDEBUG
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/BasicBlock.cpp b/llvm/lib/SandboxIR/BasicBlock.cpp
new file mode 100644
index 0000000000000..ebca41aa39da8
--- /dev/null
+++ b/llvm/lib/SandboxIR/BasicBlock.cpp
@@ -0,0 +1,163 @@
+//===- BasicBlock.cpp - The BasicBlock class of Sandbox IR ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Instruction.h"
+
+namespace llvm::sandboxir {
+
+BBIterator &BBIterator::operator++() {
+  auto ItE = BB->end();
+  assert(It != ItE && "Already at end!");
+  ++It;
+  if (It == ItE)
+    return *this;
+  Instruction &NextI = *cast<sandboxir::Instruction>(Ctx->getValue(&*It));
+  unsigned Num = NextI.getNumOfIRInstrs();
+  assert(Num > 0 && "Bad getNumOfIRInstrs()");
+  It = std::next(It, Num - 1);
+  return *this;
+}
+
+BBIterator &BBIterator::operator--() {
+  assert(It != BB->begin() && "Already at begin!");
+  if (It == BB->end()) {
+    --It;
+    return *this;
+  }
+  Instruction &CurrI = **this;
+  unsigned Num = CurrI.getNumOfIRInstrs();
+  assert(Num > 0 && "Bad getNumOfIRInstrs()");
+  assert(std::prev(It, Num - 1) != BB->begin() && "Already at begin!");
+  It = std::prev(It, Num);
+  return *this;
+}
+
+BasicBlock *BBIterator::getNodeParent() const {
+  llvm::BasicBlock *Parent = const_cast<BBIterator *>(this)->It.getNodeParent();
+  return cast<BasicBlock>(Ctx->getValue(Parent));
+}
+
+BasicBlock::iterator::pointer
+BasicBlock::iterator::getInstr(llvm::BasicBlock::iterator It) const {
+  return cast_or_null<Instruction>(Ctx->getValue(&*It));
+}
+
+Function *BasicBlock::getParent() const {
+  auto *BB = cast<llvm::BasicBlock>(Val);
+  auto *F = BB->getParent();
+  if (F == nullptr)
+    // Detached
+    return nullptr;
+  return cast_or_null<Function>(Ctx.getValue(F));
+}
+
+void BasicBlock::buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB) {
+  for (llvm::Instruction &IRef : reverse(*LLVMBB)) {
+    llvm::Instruction *I = &IRef;
+    Ctx.getOrCreateValue(I);
+    for (auto [OpIdx, Op] : enumerate(I->operands())) {
+      // Skip instruction's label operands
+      if (isa<llvm::BasicBlock>(Op))
+        continue;
+      // Skip metadata
+      if (isa<llvm::MetadataAsValue>(Op))
+        continue;
+      // Skip asm
+      if (isa<llvm::InlineAsm>(Op))
+        continue;
+      Ctx.getOrCreateValue(Op);
+    }
+  }
+#if !defined(NDEBUG)
+  verify();
+#endif
+}
+
+BasicBlock::iterator BasicBlock::begin() const {
+  llvm::BasicBlock *BB = cast<llvm::BasicBlock>(Val);
+  llvm::BasicBlock::iterator It = BB->begin();
+  if (!BB->empty()) {
+    auto *V = Ctx.getValue(&*BB->begin());
+    assert(V != nullptr && "No SandboxIR for BB->begin()!");
+    auto *I = cast<Instruction>(V);
+    unsigned Num = I->getNumOfIRInstrs();
+    assert(Num >= 1u && "Bad getNumOfIRInstrs()");
+    It = std::next(It, Num - 1);
+  }
+  return iterator(BB, It, &Ctx);
+}
+
+Instruction *BasicBlock::getTerminator() const {
+  auto *TerminatorV =
+      Ctx.getValue(cast<llvm::BasicBlock>(Val)->getTerminator());
+  return cast_or_null<Instruction>(TerminatorV);
+}
+
+Instruction &BasicBlock::front() const {
+  auto *BB = cast<llvm::BasicBlock>(Val);
+  assert(!BB->empty() && "Empty block!");
+  auto *SBI = cast<Instruction>(getContext().getValue(&*BB->begin()));
+  assert(SBI != nullptr && "Expected Instr!");
+  return *SBI;
+}
+
+Instruction &BasicBlock::back() const {
+  auto *BB = cast<llvm::BasicBlock>(Val);
+  assert(!BB->empty() && "Empty block!");
+  auto *SBI = cast<Instruction>(getContext().getValue(&*BB->rbegin()));
+  assert(SBI != nullptr && "Expected Instr!");
+  return *SBI;
+}
+
+#ifndef NDEBUG
+void BasicBlock::dumpOS(raw_ostream &OS) const {
+  llvm::BasicBlock *BB = cast<llvm::BasicBlock>(Val);
+  const auto &Name = BB->getName();
+  OS << Name;
+  if (!Name.empty())
+    OS << ":\n";
+  // If there are Instructions in the BB that are not mapped to SandboxIR, then
+  // use a crash-proof dump.
+  if (any_of(*BB, [this](llvm::Instruction &I) {
+        return Ctx.getValue(&I) == nullptr;
+      })) {
+    OS << "<Crash-proof mode!>\n";
+    DenseSet<Instruction *> Visited;
+    for (llvm::Instruction &IRef : *BB) {
+      Value *SBV = Ctx.getValue(&IRef);
+      if (SBV == nullptr)
+        OS << IRef << " *** No SandboxIR ***\n";
+      else {
+        auto *SBI = dyn_cast<Instruction>(SBV);
+        if (SBI == nullptr) {
+          OS << IRef << " *** Not a SBInstruction!!! ***\n";
+        } else {
+          if (Visited.insert(SBI).second)
+            OS << *SBI << "\n";
+        }
+      }
+    }
+  } else {
+    for (auto &SBI : *this) {
+      SBI.dumpOS(OS);
+      OS << "\n";
+    }
+  }
+}
+
+void BasicBlock::verify() const {
+  assert(isa<llvm::BasicBlock>(Val) && "Expected BasicBlock!");
+  for (const auto &I : *this) {
+    I.verify();
+  }
+}
+#endif // NDEBUG
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/CMakeLists.txt b/llvm/lib/SandboxIR/CMakeLists.txt
index b2e6f6285fea5..7c817ed133ed8 100644
--- a/llvm/lib/SandboxIR/CMakeLists.txt
+++ b/llvm/lib/SandboxIR/CMakeLists.txt
@@ -1,9 +1,18 @@
 add_llvm_component_library(LLVMSandboxIR
+  Argument.cpp
+  BasicBlock.cpp
+  Constant.cpp
+  Context.cpp
+  Instruction.cpp
+  Module.cpp
   Pass.cpp
   PassManager.cpp
+  Region.cpp
   SandboxIR.cpp
   Tracker.cpp
   Type.cpp
+  User.cpp
+  Value.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms/SandboxIR
diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp
new file mode 100644
index 0000000000000..6f1eb1e74347d
--- /dev/null
+++ b/llvm/lib/SandboxIR/Constant.cpp
@@ -0,0 +1,510 @@
+//===- Constant.cpp - The Constant classes of Sandbox IR ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Argument.h"
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Context.h"
+
+namespace llvm::sandboxir {
+
+#ifndef NDEBUG
+void Constant::dumpOS(raw_ostream &OS) const {
+  dumpCommonPrefix(OS);
+  dumpCommonSuffix(OS);
+}
+#endif // NDEBUG
+
+ConstantInt *ConstantInt::getTrue(Context &Ctx) {
+  auto *LLVMC = llvm::ConstantInt::getTrue(Ctx.LLVMCtx);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+ConstantInt *ConstantInt::getFalse(Context &Ctx) {
+  auto *LLVMC = llvm::ConstantInt::getFalse(Ctx.LLVMCtx);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+ConstantInt *ConstantInt::getBool(Context &Ctx, bool V) {
+  auto *LLVMC = llvm::ConstantInt::getBool(Ctx.LLVMCtx, V);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+Constant *ConstantInt::getTrue(Type *Ty) {
+  auto *LLVMC = llvm::ConstantInt::getTrue(Ty->LLVMTy);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+Constant *ConstantInt::getFalse(Type *Ty) {
+  auto *LLVMC = llvm::ConstantInt::getFalse(Ty->LLVMTy);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+Constant *ConstantInt::getBool(Type *Ty, bool V) {
+  auto *LLVMC = llvm::ConstantInt::getBool(Ty->LLVMTy, V);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+ConstantInt *ConstantInt::get(Type *Ty, uint64_t V, bool IsSigned) {
+  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V, IsSigned);
+  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool IsSigned) {
+  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V, IsSigned);
+  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+ConstantInt *ConstantInt::getSigned(IntegerType *Ty, int64_t V) {
+  auto *LLVMC =
+      llvm::ConstantInt::getSigned(cast<llvm::IntegerType>(Ty->LLVMTy), V);
+  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantInt::getSigned(Type *Ty, int64_t V) {
+  auto *LLVMC = llvm::ConstantInt::getSigned(Ty->LLVMTy, V);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+ConstantInt *ConstantInt::get(Context &Ctx, const APInt &V) {
+  auto *LLVMC = llvm::ConstantInt::get(Ctx.LLVMCtx, V);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+ConstantInt *ConstantInt::get(IntegerType *Ty, StringRef Str, uint8_t Radix) {
+  auto *LLVMC =
+      llvm::ConstantInt::get(cast<llvm::IntegerType>(Ty->LLVMTy), Str, Radix);
+  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantInt::get(Type *Ty, const APInt &V) {
+  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+IntegerType *ConstantInt::getIntegerType() const {
+  auto *LLVMTy = cast<llvm::ConstantInt>(Val)->getIntegerType();
+  return cast<IntegerType>(Ctx.getType(LLVMTy));
+}
+
+bool ConstantInt::isValueValidForType(Type *Ty, uint64_t V) {
+  return llvm::ConstantInt::isValueValidForType(Ty->LLVMTy, V);
+}
+bool ConstantInt::isValueValidForType(Type *Ty, int64_t V) {
+  return llvm::ConstantInt::isValueValidForType(Ty->LLVMTy, V);
+}
+
+Constant *ConstantFP::get(Type *Ty, double V) {
+  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, V);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+
+Constant *ConstantFP::get(Type *Ty, const APFloat &V) {
+  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, V);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+
+Constant *ConstantFP::get(Type *Ty, StringRef Str) {
+  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, Str);
+  return Ty->getContext().getOrCreateConstant(LLVMC);
+}
+
+ConstantFP *ConstantFP::get(const APFloat &V, Context &Ctx) {
+  auto *LLVMC = llvm::ConstantFP::get(Ctx.LLVMCtx, V);
+  return cast<ConstantFP>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
+  auto *LLVMC = llvm::ConstantFP::getNaN(Ty->LLVMTy, Negative, Payload);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
+  auto *LLVMC = llvm::ConstantFP::getQNaN(Ty->LLVMTy, Negative, Payload);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
+  auto *LLVMC = llvm::ConstantFP::getSNaN(Ty->LLVMTy, Negative, Payload);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantFP::getZero(Type *Ty, bool Negative) {
+  auto *LLVMC = llvm::ConstantFP::getZero(Ty->LLVMTy, Negative);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantFP::getNegativeZero(Type *Ty) {
+  auto *LLVMC = llvm::ConstantFP::getNegativeZero(Ty->LLVMTy);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
+  auto *LLVMC = llvm::ConstantFP::getInfinity(Ty->LLVMTy, Negative);
+  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+bool ConstantFP::isValueValidForType(Type *Ty, const APFloat &V) {
+  return llvm::ConstantFP::isValueValidForType(Ty->LLVMTy, V);
+}
+
+Constant *ConstantArray::get(ArrayType *T, ArrayRef<Constant *> V) {
+  auto &Ctx = T->getContext();
+  SmallVector<llvm::Constant *> LLVMValues;
+  LLVMValues.reserve(V.size());
+  for (auto *Elm : V)
+    LLVMValues.push_back(cast<llvm::Constant>(Elm->Val));
+  auto *LLVMC =
+      llvm::ConstantArray::get(cast<llvm::ArrayType>(T->LLVMTy), LLVMValues);
+  return cast<ConstantArray>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+ArrayType *ConstantArray::getType() const {
+  return cast<ArrayType>(
+      Ctx.getType(cast<llvm::ConstantArray>(Val)->getType()));
+}
+
+Constant *ConstantStruct::get(StructType *T, ArrayRef<Constant *> V) {
+  auto &Ctx = T->getContext();
+  SmallVector<llvm::Constant *> LLVMValues;
+  LLVMValues.reserve(V.size());
+  for (auto *Elm : V)
+    LLVMValues.push_back(cast<llvm::Constant>(Elm->Val));
+  auto *LLVMC =
+      llvm::ConstantStruct::get(cast<llvm::StructType>(T->LLVMTy), LLVMValues);
+  return cast<ConstantStruct>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+StructType *ConstantStruct::getTypeForElements(Context &Ctx,
+                                               ArrayRef<Constant *> V,
+                                               bool Packed) {
+  unsigned VecSize = V.size();
+  SmallVector<Type *, 16> EltTypes;
+  EltTypes.reserve(VecSize);
+  for (Constant *Elm : V)
+    EltTypes.push_back(Elm->getType());
+  return StructType::get(Ctx, EltTypes, Packed);
+}
+
+ConstantAggregateZero *ConstantAggregateZero::get(Type *Ty) {
+  auto *LLVMC = llvm::ConstantAggregateZero::get(Ty->LLVMTy);
+  return cast<ConstantAggregateZero>(
+      Ty->getContext().getOrCreateConstant(LLVMC));
+}
+
+Constant *ConstantAggregateZero::getSequentialElement() const {
+  return cast<Constant>(Ctx.getValue(
+      cast<llvm::ConstantAggregateZero>(Val)->getSequentialElement()));
+}
+Constant *ConstantAggregateZero::getStructElement(unsigned Elt) const {
+  return cast<Constant>(Ctx.getValue(
+      cast<llvm::ConstantAggregateZero>(Val)->getStructElement(Elt)));
+}
+Constant *ConstantAggregateZero::getElementValue(Constant *C) const {
+  return cast<Constant>(
+      Ctx.getValue(cast<llvm::ConstantAggregateZero>(Val)->getElementValue(
+          cast<llvm::Constant>(C->Val))));
+}
+Constant *ConstantAggregateZero::getElementValue(unsigned Idx) const {
+  return cast<Constant>(Ctx.getValue(
+      cast<llvm::ConstantAggregateZero>(Val)->getElementValue(Idx)));
+}
+
+ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) {
+  auto *LLVMC =
+      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty->LLVMTy));
+  return cast<ConstantPointerNull>(Ty->getContext().getOrCreateConstant(LLVMC));
+}
+
+PointerType *ConstantPointerNull::getType() const {
+  return cast<PointerType>(
+      Ctx.getType(cast<llvm::ConstantPointerNull>(Val)->getType()));
+}
+
+UndefValue *UndefValue::get(Type *T) {
+  auto *LLVMC = llvm::UndefValue::get(T->LLVMTy);
+  return cast<UndefValue>(T->getContext().getOrCreateConstant(LLVMC));
+}
+
+UndefValue *UndefValue::getSequentialElement() const {
+  return cast<UndefValue>(Ctx.getOrCreateConstant(
+      cast<llvm::UndefValue>(Val)->getSequentialElement()));
+}
+
+UndefValue *UndefValue::getStructElement(unsigned Elt) const {
+  return cast<UndefValue>(Ctx.getOrCreateConstant(
+      cast<llvm::UndefValue>(Val)->getStructElement(Elt)));
+}
+
+UndefValue *UndefValue::getElementValue(Constant *C) const {
+  return cast<UndefValue>(
+      Ctx.getOrCreateConstant(cast<llvm::UndefValue>(Val)->getElementValue(
+          cast<llvm::Constant>(C->Val))));
+}
+
+UndefValue *UndefValue::getElementValue(unsigned Idx) const {
+  return cast<UndefValue>(Ctx.getOrCreateConstant(
+      cast<llvm::UndefValue>(Val)->getElementValue(Idx)));
+}
+
+PoisonValue *PoisonValue::get(Type *T) {
+  auto *LLVMC = llvm::PoisonValue::get(T->LLVMTy);
+  return cast<PoisonValue>(T->getContext().getOrCreateConstant(LLVMC));
+}
+
+PoisonValue *PoisonValue::getSequentialElement() const {
+  return cast<PoisonValue>(Ctx.getOrCreateConstant(
+      cast<llvm::PoisonValue>(Val)->getSequentialElement()));
+}
+
+PoisonValue *PoisonValue::getStructElement(unsigned Elt) const {
+  return cast<PoisonValue>(Ctx.getOrCreateConstant(
+      cast<llvm::PoisonValue>(Val)->getStructElement(Elt)));
+}
+
+PoisonValue *PoisonValue::getElementValue(Constant *C) const {
+  return cast<PoisonValue>(
+      Ctx.getOrCreateConstant(cast<llvm::PoisonValue>(Val)->getElementValue(
+          cast<llvm::Constant>(C->Val))));
+}
+
+PoisonValue *PoisonValue::getElementValue(unsigned Idx) const {
+  return cast<PoisonValue>(Ctx.getOrCreateConstant(
+      cast<llvm::PoisonValue>(Val)->getElementValue(Idx)));
+}
+
+void GlobalObject::setAlignment(MaybeAlign Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalObject::getAlign, &GlobalObject::setAlignment>>(
+          this);
+  cast<llvm::GlobalObject>(Val)->setAlignment(Align);
+}
+
+void GlobalObject::setGlobalObjectSubClassData(unsigned V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalObject::getGlobalObjectSubClassData,
+                        &GlobalObject::setGlobalObjectSubClassData>>(this);
+  cast<llvm::GlobalObject>(Val)->setGlobalObjectSubClassData(V);
+}
+
+void GlobalObject::setSection(StringRef S) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalObject::getSection, &GlobalObject::setSection>>(
+          this);
+  cast<llvm::GlobalObject>(Val)->setSection(S);
+}
+
+template <typename GlobalT, typename LLVMGlobalT, typename ParentT,
+          typename LLVMParentT>
+GlobalT &GlobalWithNodeAPI<GlobalT, LLVMGlobalT, ParentT, LLVMParentT>::
+    LLVMGVToGV::operator()(LLVMGlobalT &LLVMGV) const {
+  return cast<GlobalT>(*Ctx.getValue(&LLVMGV));
+}
+
+// Explicit instantiations.
+template class GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
+                                 llvm::GlobalObject>;
+template class GlobalWithNodeAPI<Function, llvm::Function, GlobalObject,
+                                 llvm::GlobalObject>;
+template class GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
+                                 GlobalObject, llvm::GlobalObject>;
+template class GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
+                                 llvm::GlobalValue>;
+
+void GlobalIFunc::setResolver(Constant *Resolver) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalIFunc::getResolver, &GlobalIFunc::setResolver>>(
+          this);
+  cast<llvm::GlobalIFunc>(Val)->setResolver(
+      cast<llvm::Constant>(Resolver->Val));
+}
+
+Constant *GlobalIFunc::getResolver() const {
+  return Ctx.getOrCreateConstant(cast<llvm::GlobalIFunc>(Val)->getResolver());
+}
+
+Function *GlobalIFunc::getResolverFunction() {
+  return cast<Function>(Ctx.getOrCreateConstant(
+      cast<llvm::GlobalIFunc>(Val)->getResolverFunction()));
+}
+
+GlobalVariable &
+GlobalVariable::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV) const {
+  return cast<GlobalVariable>(*Ctx.getValue(&LLVMGV));
+}
+
+Constant *GlobalVariable::getInitializer() const {
+  return Ctx.getOrCreateConstant(
+      cast<llvm::GlobalVariable>(Val)->getInitializer());
+}
+
+void GlobalVariable::setInitializer(Constant *InitVal) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalVariable::getInitializer,
+                                       &GlobalVariable::setInitializer>>(this);
+  cast<llvm::GlobalVariable>(Val)->setInitializer(
+      cast<llvm::Constant>(InitVal->Val));
+}
+
+void GlobalVariable::setConstant(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalVariable::isConstant,
+                                       &GlobalVariable::setConstant>>(this);
+  cast<llvm::GlobalVariable>(Val)->setConstant(V);
+}
+
+void GlobalVariable::setExternallyInitialized(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalVariable::isExternallyInitialized,
+                        &GlobalVariable::setExternallyInitialized>>(this);
+  cast<llvm::GlobalVariable>(Val)->setExternallyInitialized(V);
+}
+
+void GlobalAlias::setAliasee(Constant *Aliasee) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&GlobalAlias::getAliasee, &GlobalAlias::setAliasee>>(
+          this);
+  cast<llvm::GlobalAlias>(Val)->setAliasee(cast<llvm::Constant>(Aliasee->Val));
+}
+
+Constant *GlobalAlias::getAliasee() const {
+  return cast<Constant>(
+      Ctx.getOrCreateConstant(cast<llvm::GlobalAlias>(Val)->getAliasee()));
+}
+
+const GlobalObject *GlobalAlias::getAliaseeObject() const {
+  return cast<GlobalObject>(Ctx.getOrCreateConstant(
+      cast<llvm::GlobalAlias>(Val)->getAliaseeObject()));
+}
+
+void GlobalValue::setUnnamedAddr(UnnamedAddr V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalValue::getUnnamedAddr,
+                                       &GlobalValue::setUnnamedAddr>>(this);
+  cast<llvm::GlobalValue>(Val)->setUnnamedAddr(V);
+}
+
+void GlobalValue::setVisibility(VisibilityTypes V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&GlobalValue::getVisibility,
+                                       &GlobalValue::setVisibility>>(this);
+  cast<llvm::GlobalValue>(Val)->setVisibility(V);
+}
+
+NoCFIValue *NoCFIValue::get(GlobalValue *GV) {
+  auto *LLVMC = llvm::NoCFIValue::get(cast<llvm::GlobalValue>(GV->Val));
+  return cast<NoCFIValue>(GV->getContext().getOrCreateConstant(LLVMC));
+}
+
+GlobalValue *NoCFIValue::getGlobalValue() const {
+  auto *LLVMC = cast<llvm::NoCFIValue>(Val)->getGlobalValue();
+  return cast<GlobalValue>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+PointerType *NoCFIValue::getType() const {
+  return cast<PointerType>(Ctx.getType(cast<llvm::NoCFIValue>(Val)->getType()));
+}
+
+ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key,
+                                      ConstantInt *Disc, Constant *AddrDisc) {
+  auto *LLVMC = llvm::ConstantPtrAuth::get(
+      cast<llvm::Constant>(Ptr->Val), cast<llvm::ConstantInt>(Key->Val),
+      cast<llvm::ConstantInt>(Disc->Val), cast<llvm::Constant>(AddrDisc->Val));
+  return cast<ConstantPtrAuth>(Ptr->getContext().getOrCreateConstant(LLVMC));
+}
+
+Constant *ConstantPtrAuth::getPointer() const {
+  return Ctx.getOrCreateConstant(
+      cast<llvm::ConstantPtrAuth>(Val)->getPointer());
+}
+
+ConstantInt *ConstantPtrAuth::getKey() const {
+  return cast<ConstantInt>(
+      Ctx.getOrCreateConstant(cast<llvm::ConstantPtrAuth>(Val)->getKey()));
+}
+
+ConstantInt *ConstantPtrAuth::getDiscriminator() const {
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(
+      cast<llvm::ConstantPtrAuth>(Val)->getDiscriminator()));
+}
+
+Constant *ConstantPtrAuth::getAddrDiscriminator() const {
+  return Ctx.getOrCreateConstant(
+      cast<llvm::ConstantPtrAuth>(Val)->getAddrDiscriminator());
+}
+
+ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const {
+  auto *LLVMC = cast<llvm::ConstantPtrAuth>(Val)->getWithSameSchema(
+      cast<llvm::Constant>(Pointer->Val));
+  return cast<ConstantPtrAuth>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+BlockAddress *BlockAddress::get(Function *F, BasicBlock *BB) {
+  auto *LLVMC = llvm::BlockAddress::get(cast<llvm::Function>(F->Val),
+                                        cast<llvm::BasicBlock>(BB->Val));
+  return cast<BlockAddress>(F->getContext().getOrCreateConstant(LLVMC));
+}
+
+BlockAddress *BlockAddress::get(BasicBlock *BB) {
+  auto *LLVMC = llvm::BlockAddress::get(cast<llvm::BasicBlock>(BB->Val));
+  return cast<BlockAddress>(BB->getContext().getOrCreateConstant(LLVMC));
+}
+
+BlockAddress *BlockAddress::lookup(const BasicBlock *BB) {
+  auto *LLVMC = llvm::BlockAddress::lookup(cast<llvm::BasicBlock>(BB->Val));
+  return cast_or_null<BlockAddress>(BB->getContext().getValue(LLVMC));
+}
+
+Function *BlockAddress::getFunction() const {
+  return cast<Function>(
+      Ctx.getValue(cast<llvm::BlockAddress>(Val)->getFunction()));
+}
+
+BasicBlock *BlockAddress::getBasicBlock() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::BlockAddress>(Val)->getBasicBlock()));
+}
+
+DSOLocalEquivalent *DSOLocalEquivalent::get(GlobalValue *GV) {
+  auto *LLVMC = llvm::DSOLocalEquivalent::get(cast<llvm::GlobalValue>(GV->Val));
+  return cast<DSOLocalEquivalent>(GV->getContext().getValue(LLVMC));
+}
+
+GlobalValue *DSOLocalEquivalent::getGlobalValue() const {
+  return cast<GlobalValue>(
+      Ctx.getValue(cast<llvm::DSOLocalEquivalent>(Val)->getGlobalValue()));
+}
+
+FunctionType *Function::getFunctionType() const {
+  return cast<FunctionType>(
+      Ctx.getType(cast<llvm::Function>(Val)->getFunctionType()));
+}
+
+#ifndef NDEBUG
+void Function::dumpNameAndArgs(raw_ostream &OS) const {
+  auto *F = cast<llvm::Function>(Val);
+  OS << *F->getReturnType() << " @" << F->getName() << "(";
+  interleave(
+      F->args(),
+      [this, &OS](const llvm::Argument &LLVMArg) {
+        auto *SBArg = cast_or_null<Argument>(Ctx.getValue(&LLVMArg));
+        if (SBArg == nullptr)
+          OS << "NULL";
+        else
+          SBArg->printAsOperand(OS);
+      },
+      [&] { OS << ", "; });
+  OS << ")";
+}
+
+void Function::dumpOS(raw_ostream &OS) const {
+  dumpNameAndArgs(OS);
+  OS << " {\n";
+  auto *LLVMF = cast<llvm::Function>(Val);
+  interleave(
+      *LLVMF,
+      [this, &OS](const llvm::BasicBlock &LLVMBB) {
+        auto *BB = cast_or_null<BasicBlock>(Ctx.getValue(&LLVMBB));
+        if (BB == nullptr)
+          OS << "NULL";
+        else
+          OS << *BB;
+      },
+      [&OS] { OS << "\n"; });
+  OS << "}\n";
+}
+#endif // NDEBUG
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Context.cpp b/llvm/lib/SandboxIR/Context.cpp
new file mode 100644
index 0000000000000..0a61e329b78c5
--- /dev/null
+++ b/llvm/lib/SandboxIR/Context.cpp
@@ -0,0 +1,728 @@
+//===- Context.cpp - The Context class of Sandbox IR ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Module.h"
+
+namespace llvm::sandboxir {
+
+std::unique_ptr<Value> Context::detachLLVMValue(llvm::Value *V) {
+  std::unique_ptr<Value> Erased;
+  auto It = LLVMValueToValueMap.find(V);
+  if (It != LLVMValueToValueMap.end()) {
+    auto *Val = It->second.release();
+    Erased = std::unique_ptr<Value>(Val);
+    LLVMValueToValueMap.erase(It);
+  }
+  return Erased;
+}
+
+std::unique_ptr<Value> Context::detach(Value *V) {
+  assert(V->getSubclassID() != Value::ClassID::Constant &&
+         "Can't detach a constant!");
+  assert(V->getSubclassID() != Value::ClassID::User && "Can't detach a user!");
+  return detachLLVMValue(V->Val);
+}
+
+Value *Context::registerValue(std::unique_ptr<Value> &&VPtr) {
+  assert(VPtr->getSubclassID() != Value::ClassID::User &&
+         "Can't register a user!");
+
+  // Track creation of instructions.
+  // Please note that we don't allow the creation of detached instructions,
+  // meaning that the instructions need to be inserted into a block upon
+  // creation. This is why the tracker class combines creation and insertion.
+  if (auto *I = dyn_cast<Instruction>(VPtr.get()))
+    getTracker().emplaceIfTracking<CreateAndInsertInst>(I);
+
+  Value *V = VPtr.get();
+  [[maybe_unused]] auto Pair =
+      LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
+  assert(Pair.second && "Already exists!");
+  return V;
+}
+
+Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
+  auto Pair = LLVMValueToValueMap.insert({LLVMV, nullptr});
+  auto It = Pair.first;
+  if (!Pair.second)
+    return It->second.get();
+
+  if (auto *C = dyn_cast<llvm::Constant>(LLVMV)) {
+    switch (C->getValueID()) {
+    case llvm::Value::ConstantIntVal:
+      It->second = std::unique_ptr<ConstantInt>(
+          new ConstantInt(cast<llvm::ConstantInt>(C), *this));
+      return It->second.get();
+    case llvm::Value::ConstantFPVal:
+      It->second = std::unique_ptr<ConstantFP>(
+          new ConstantFP(cast<llvm::ConstantFP>(C), *this));
+      return It->second.get();
+    case llvm::Value::BlockAddressVal:
+      It->second = std::unique_ptr<BlockAddress>(
+          new BlockAddress(cast<llvm::BlockAddress>(C), *this));
+      return It->second.get();
+    case llvm::Value::ConstantTokenNoneVal:
+      It->second = std::unique_ptr<ConstantTokenNone>(
+          new ConstantTokenNone(cast<llvm::ConstantTokenNone>(C), *this));
+      return It->second.get();
+    case llvm::Value::ConstantAggregateZeroVal: {
+      auto *CAZ = cast<llvm::ConstantAggregateZero>(C);
+      It->second = std::unique_ptr<ConstantAggregateZero>(
+          new ConstantAggregateZero(CAZ, *this));
+      auto *Ret = It->second.get();
+      // Must create sandboxir for elements.
+      auto EC = CAZ->getElementCount();
+      if (EC.isFixed()) {
+        for (auto ElmIdx : seq<unsigned>(0, EC.getFixedValue()))
+          getOrCreateValueInternal(CAZ->getElementValue(ElmIdx), CAZ);
+      }
+      return Ret;
+    }
+    case llvm::Value::ConstantPointerNullVal:
+      It->second = std::unique_ptr<ConstantPointerNull>(
+          new ConstantPointerNull(cast<llvm::ConstantPointerNull>(C), *this));
+      return It->second.get();
+    case llvm::Value::PoisonValueVal:
+      It->second = std::unique_ptr<PoisonValue>(
+          new PoisonValue(cast<llvm::PoisonValue>(C), *this));
+      return It->second.get();
+    case llvm::Value::UndefValueVal:
+      It->second = std::unique_ptr<UndefValue>(
+          new UndefValue(cast<llvm::UndefValue>(C), *this));
+      return It->second.get();
+    case llvm::Value::DSOLocalEquivalentVal: {
+      auto *DSOLE = cast<llvm::DSOLocalEquivalent>(C);
+      It->second = std::unique_ptr<DSOLocalEquivalent>(
+          new DSOLocalEquivalent(DSOLE, *this));
+      auto *Ret = It->second.get();
+      getOrCreateValueInternal(DSOLE->getGlobalValue(), DSOLE);
+      return Ret;
+    }
+    case llvm::Value::ConstantArrayVal:
+      It->second = std::unique_ptr<ConstantArray>(
+          new ConstantArray(cast<llvm::ConstantArray>(C), *this));
+      break;
+    case llvm::Value::ConstantStructVal:
+      It->second = std::unique_ptr<ConstantStruct>(
+          new ConstantStruct(cast<llvm::ConstantStruct>(C), *this));
+      break;
+    case llvm::Value::ConstantVectorVal:
+      It->second = std::unique_ptr<ConstantVector>(
+          new ConstantVector(cast<llvm::ConstantVector>(C), *this));
+      break;
+    case llvm::Value::FunctionVal:
+      It->second = std::unique_ptr<Function>(
+          new Function(cast<llvm::Function>(C), *this));
+      break;
+    case llvm::Value::GlobalIFuncVal:
+      It->second = std::unique_ptr<GlobalIFunc>(
+          new GlobalIFunc(cast<llvm::GlobalIFunc>(C), *this));
+      break;
+    case llvm::Value::GlobalVariableVal:
+      It->second = std::unique_ptr<GlobalVariable>(
+          new GlobalVariable(cast<llvm::GlobalVariable>(C), *this));
+      break;
+    case llvm::Value::GlobalAliasVal:
+      It->second = std::unique_ptr<GlobalAlias>(
+          new GlobalAlias(cast<llvm::GlobalAlias>(C), *this));
+      break;
+    case llvm::Value::NoCFIValueVal:
+      It->second = std::unique_ptr<NoCFIValue>(
+          new NoCFIValue(cast<llvm::NoCFIValue>(C), *this));
+      break;
+    case llvm::Value::ConstantPtrAuthVal:
+      It->second = std::unique_ptr<ConstantPtrAuth>(
+          new ConstantPtrAuth(cast<llvm::ConstantPtrAuth>(C), *this));
+      break;
+    case llvm::Value::ConstantExprVal:
+      It->second = std::unique_ptr<ConstantExpr>(
+          new ConstantExpr(cast<llvm::ConstantExpr>(C), *this));
+      break;
+    default:
+      It->second = std::unique_ptr<Constant>(new Constant(C, *this));
+      break;
+    }
+    auto *NewC = It->second.get();
+    for (llvm::Value *COp : C->operands())
+      getOrCreateValueInternal(COp, C);
+    return NewC;
+  }
+  if (auto *Arg = dyn_cast<llvm::Argument>(LLVMV)) {
+    It->second = std::unique_ptr<Argument>(new Argument(Arg, *this));
+    return It->second.get();
+  }
+  if (auto *BB = dyn_cast<llvm::BasicBlock>(LLVMV)) {
+    assert(isa<llvm::BlockAddress>(U) &&
+           "This won't create a SBBB, don't call this function directly!");
+    if (auto *SBBB = getValue(BB))
+      return SBBB;
+    return nullptr;
+  }
+  assert(isa<llvm::Instruction>(LLVMV) && "Expected Instruction");
+
+  switch (cast<llvm::Instruction>(LLVMV)->getOpcode()) {
+  case llvm::Instruction::VAArg: {
+    auto *LLVMVAArg = cast<llvm::VAArgInst>(LLVMV);
+    It->second = std::unique_ptr<VAArgInst>(new VAArgInst(LLVMVAArg, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Freeze: {
+    auto *LLVMFreeze = cast<llvm::FreezeInst>(LLVMV);
+    It->second = std::unique_ptr<FreezeInst>(new FreezeInst(LLVMFreeze, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Fence: {
+    auto *LLVMFence = cast<llvm::FenceInst>(LLVMV);
+    It->second = std::unique_ptr<FenceInst>(new FenceInst(LLVMFence, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Select: {
+    auto *LLVMSel = cast<llvm::SelectInst>(LLVMV);
+    It->second = std::unique_ptr<SelectInst>(new SelectInst(LLVMSel, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::ExtractElement: {
+    auto *LLVMIns = cast<llvm::ExtractElementInst>(LLVMV);
+    It->second = std::unique_ptr<ExtractElementInst>(
+        new ExtractElementInst(LLVMIns, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::InsertElement: {
+    auto *LLVMIns = cast<llvm::InsertElementInst>(LLVMV);
+    It->second = std::unique_ptr<InsertElementInst>(
+        new InsertElementInst(LLVMIns, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::ShuffleVector: {
+    auto *LLVMIns = cast<llvm::ShuffleVectorInst>(LLVMV);
+    It->second = std::unique_ptr<ShuffleVectorInst>(
+        new ShuffleVectorInst(LLVMIns, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::ExtractValue: {
+    auto *LLVMIns = cast<llvm::ExtractValueInst>(LLVMV);
+    It->second =
+        std::unique_ptr<ExtractValueInst>(new ExtractValueInst(LLVMIns, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::InsertValue: {
+    auto *LLVMIns = cast<llvm::InsertValueInst>(LLVMV);
+    It->second =
+        std::unique_ptr<InsertValueInst>(new InsertValueInst(LLVMIns, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Br: {
+    auto *LLVMBr = cast<llvm::BranchInst>(LLVMV);
+    It->second = std::unique_ptr<BranchInst>(new BranchInst(LLVMBr, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Load: {
+    auto *LLVMLd = cast<llvm::LoadInst>(LLVMV);
+    It->second = std::unique_ptr<LoadInst>(new LoadInst(LLVMLd, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Store: {
+    auto *LLVMSt = cast<llvm::StoreInst>(LLVMV);
+    It->second = std::unique_ptr<StoreInst>(new StoreInst(LLVMSt, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Ret: {
+    auto *LLVMRet = cast<llvm::ReturnInst>(LLVMV);
+    It->second = std::unique_ptr<ReturnInst>(new ReturnInst(LLVMRet, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Call: {
+    auto *LLVMCall = cast<llvm::CallInst>(LLVMV);
+    It->second = std::unique_ptr<CallInst>(new CallInst(LLVMCall, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Invoke: {
+    auto *LLVMInvoke = cast<llvm::InvokeInst>(LLVMV);
+    It->second = std::unique_ptr<InvokeInst>(new InvokeInst(LLVMInvoke, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CallBr: {
+    auto *LLVMCallBr = cast<llvm::CallBrInst>(LLVMV);
+    It->second = std::unique_ptr<CallBrInst>(new CallBrInst(LLVMCallBr, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::LandingPad: {
+    auto *LLVMLPad = cast<llvm::LandingPadInst>(LLVMV);
+    It->second =
+        std::unique_ptr<LandingPadInst>(new LandingPadInst(LLVMLPad, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CatchPad: {
+    auto *LLVMCPI = cast<llvm::CatchPadInst>(LLVMV);
+    It->second =
+        std::unique_ptr<CatchPadInst>(new CatchPadInst(LLVMCPI, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CleanupPad: {
+    auto *LLVMCPI = cast<llvm::CleanupPadInst>(LLVMV);
+    It->second =
+        std::unique_ptr<CleanupPadInst>(new CleanupPadInst(LLVMCPI, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CatchRet: {
+    auto *LLVMCRI = cast<llvm::CatchReturnInst>(LLVMV);
+    It->second =
+        std::unique_ptr<CatchReturnInst>(new CatchReturnInst(LLVMCRI, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CleanupRet: {
+    auto *LLVMCRI = cast<llvm::CleanupReturnInst>(LLVMV);
+    It->second = std::unique_ptr<CleanupReturnInst>(
+        new CleanupReturnInst(LLVMCRI, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::GetElementPtr: {
+    auto *LLVMGEP = cast<llvm::GetElementPtrInst>(LLVMV);
+    It->second = std::unique_ptr<GetElementPtrInst>(
+        new GetElementPtrInst(LLVMGEP, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::CatchSwitch: {
+    auto *LLVMCatchSwitchInst = cast<llvm::CatchSwitchInst>(LLVMV);
+    It->second = std::unique_ptr<CatchSwitchInst>(
+        new CatchSwitchInst(LLVMCatchSwitchInst, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Resume: {
+    auto *LLVMResumeInst = cast<llvm::ResumeInst>(LLVMV);
+    It->second =
+        std::unique_ptr<ResumeInst>(new ResumeInst(LLVMResumeInst, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Switch: {
+    auto *LLVMSwitchInst = cast<llvm::SwitchInst>(LLVMV);
+    It->second =
+        std::unique_ptr<SwitchInst>(new SwitchInst(LLVMSwitchInst, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::FNeg: {
+    auto *LLVMUnaryOperator = cast<llvm::UnaryOperator>(LLVMV);
+    It->second = std::unique_ptr<UnaryOperator>(
+        new UnaryOperator(LLVMUnaryOperator, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Add:
+  case llvm::Instruction::FAdd:
+  case llvm::Instruction::Sub:
+  case llvm::Instruction::FSub:
+  case llvm::Instruction::Mul:
+  case llvm::Instruction::FMul:
+  case llvm::Instruction::UDiv:
+  case llvm::Instruction::SDiv:
+  case llvm::Instruction::FDiv:
+  case llvm::Instruction::URem:
+  case llvm::Instruction::SRem:
+  case llvm::Instruction::FRem:
+  case llvm::Instruction::Shl:
+  case llvm::Instruction::LShr:
+  case llvm::Instruction::AShr:
+  case llvm::Instruction::And:
+  case llvm::Instruction::Or:
+  case llvm::Instruction::Xor: {
+    auto *LLVMBinaryOperator = cast<llvm::BinaryOperator>(LLVMV);
+    It->second = std::unique_ptr<BinaryOperator>(
+        new BinaryOperator(LLVMBinaryOperator, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::AtomicRMW: {
+    auto *LLVMAtomicRMW = cast<llvm::AtomicRMWInst>(LLVMV);
+    It->second =
+        std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(LLVMAtomicRMW, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::AtomicCmpXchg: {
+    auto *LLVMAtomicCmpXchg = cast<llvm::AtomicCmpXchgInst>(LLVMV);
+    It->second = std::unique_ptr<AtomicCmpXchgInst>(
+        new AtomicCmpXchgInst(LLVMAtomicCmpXchg, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Alloca: {
+    auto *LLVMAlloca = cast<llvm::AllocaInst>(LLVMV);
+    It->second = std::unique_ptr<AllocaInst>(new AllocaInst(LLVMAlloca, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::ZExt:
+  case llvm::Instruction::SExt:
+  case llvm::Instruction::FPToUI:
+  case llvm::Instruction::FPToSI:
+  case llvm::Instruction::FPExt:
+  case llvm::Instruction::PtrToInt:
+  case llvm::Instruction::IntToPtr:
+  case llvm::Instruction::SIToFP:
+  case llvm::Instruction::UIToFP:
+  case llvm::Instruction::Trunc:
+  case llvm::Instruction::FPTrunc:
+  case llvm::Instruction::BitCast:
+  case llvm::Instruction::AddrSpaceCast: {
+    auto *LLVMCast = cast<llvm::CastInst>(LLVMV);
+    It->second = std::unique_ptr<CastInst>(new CastInst(LLVMCast, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::PHI: {
+    auto *LLVMPhi = cast<llvm::PHINode>(LLVMV);
+    It->second = std::unique_ptr<PHINode>(new PHINode(LLVMPhi, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::ICmp: {
+    auto *LLVMICmp = cast<llvm::ICmpInst>(LLVMV);
+    It->second = std::unique_ptr<ICmpInst>(new ICmpInst(LLVMICmp, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::FCmp: {
+    auto *LLVMFCmp = cast<llvm::FCmpInst>(LLVMV);
+    It->second = std::unique_ptr<FCmpInst>(new FCmpInst(LLVMFCmp, *this));
+    return It->second.get();
+  }
+  case llvm::Instruction::Unreachable: {
+    auto *LLVMUnreachable = cast<llvm::UnreachableInst>(LLVMV);
+    It->second = std::unique_ptr<UnreachableInst>(
+        new UnreachableInst(LLVMUnreachable, *this));
+    return It->second.get();
+  }
+  default:
+    break;
+  }
+
+  It->second = std::unique_ptr<OpaqueInst>(
+      new OpaqueInst(cast<llvm::Instruction>(LLVMV), *this));
+  return It->second.get();
+}
+
+Argument *Context::getOrCreateArgument(llvm::Argument *LLVMArg) {
+  auto Pair = LLVMValueToValueMap.insert({LLVMArg, nullptr});
+  auto It = Pair.first;
+  if (Pair.second) {
+    It->second = std::unique_ptr<Argument>(new Argument(LLVMArg, *this));
+    return cast<Argument>(It->second.get());
+  }
+  return cast<Argument>(It->second.get());
+}
+
+Constant *Context::getOrCreateConstant(llvm::Constant *LLVMC) {
+  return cast<Constant>(getOrCreateValueInternal(LLVMC, 0));
+}
+
+BasicBlock *Context::createBasicBlock(llvm::BasicBlock *LLVMBB) {
+  assert(getValue(LLVMBB) == nullptr && "Already exists!");
+  auto NewBBPtr = std::unique_ptr<BasicBlock>(new BasicBlock(LLVMBB, *this));
+  auto *BB = cast<BasicBlock>(registerValue(std::move(NewBBPtr)));
+  // Create SandboxIR for BB's body.
+  BB->buildBasicBlockFromLLVMIR(LLVMBB);
+  return BB;
+}
+
+VAArgInst *Context::createVAArgInst(llvm::VAArgInst *SI) {
+  auto NewPtr = std::unique_ptr<VAArgInst>(new VAArgInst(SI, *this));
+  return cast<VAArgInst>(registerValue(std::move(NewPtr)));
+}
+
+FreezeInst *Context::createFreezeInst(llvm::FreezeInst *SI) {
+  auto NewPtr = std::unique_ptr<FreezeInst>(new FreezeInst(SI, *this));
+  return cast<FreezeInst>(registerValue(std::move(NewPtr)));
+}
+
+FenceInst *Context::createFenceInst(llvm::FenceInst *SI) {
+  auto NewPtr = std::unique_ptr<FenceInst>(new FenceInst(SI, *this));
+  return cast<FenceInst>(registerValue(std::move(NewPtr)));
+}
+
+SelectInst *Context::createSelectInst(llvm::SelectInst *SI) {
+  auto NewPtr = std::unique_ptr<SelectInst>(new SelectInst(SI, *this));
+  return cast<SelectInst>(registerValue(std::move(NewPtr)));
+}
+
+ExtractElementInst *
+Context::createExtractElementInst(llvm::ExtractElementInst *EEI) {
+  auto NewPtr =
+      std::unique_ptr<ExtractElementInst>(new ExtractElementInst(EEI, *this));
+  return cast<ExtractElementInst>(registerValue(std::move(NewPtr)));
+}
+
+InsertElementInst *
+Context::createInsertElementInst(llvm::InsertElementInst *IEI) {
+  auto NewPtr =
+      std::unique_ptr<InsertElementInst>(new InsertElementInst(IEI, *this));
+  return cast<InsertElementInst>(registerValue(std::move(NewPtr)));
+}
+
+ShuffleVectorInst *
+Context::createShuffleVectorInst(llvm::ShuffleVectorInst *SVI) {
+  auto NewPtr =
+      std::unique_ptr<ShuffleVectorInst>(new ShuffleVectorInst(SVI, *this));
+  return cast<ShuffleVectorInst>(registerValue(std::move(NewPtr)));
+}
+
+ExtractValueInst *Context::createExtractValueInst(llvm::ExtractValueInst *EVI) {
+  auto NewPtr =
+      std::unique_ptr<ExtractValueInst>(new ExtractValueInst(EVI, *this));
+  return cast<ExtractValueInst>(registerValue(std::move(NewPtr)));
+}
+
+InsertValueInst *Context::createInsertValueInst(llvm::InsertValueInst *IVI) {
+  auto NewPtr =
+      std::unique_ptr<InsertValueInst>(new InsertValueInst(IVI, *this));
+  return cast<InsertValueInst>(registerValue(std::move(NewPtr)));
+}
+
+BranchInst *Context::createBranchInst(llvm::BranchInst *BI) {
+  auto NewPtr = std::unique_ptr<BranchInst>(new BranchInst(BI, *this));
+  return cast<BranchInst>(registerValue(std::move(NewPtr)));
+}
+
+LoadInst *Context::createLoadInst(llvm::LoadInst *LI) {
+  auto NewPtr = std::unique_ptr<LoadInst>(new LoadInst(LI, *this));
+  return cast<LoadInst>(registerValue(std::move(NewPtr)));
+}
+
+StoreInst *Context::createStoreInst(llvm::StoreInst *SI) {
+  auto NewPtr = std::unique_ptr<StoreInst>(new StoreInst(SI, *this));
+  return cast<StoreInst>(registerValue(std::move(NewPtr)));
+}
+
+ReturnInst *Context::createReturnInst(llvm::ReturnInst *I) {
+  auto NewPtr = std::unique_ptr<ReturnInst>(new ReturnInst(I, *this));
+  return cast<ReturnInst>(registerValue(std::move(NewPtr)));
+}
+
+CallInst *Context::createCallInst(llvm::CallInst *I) {
+  auto NewPtr = std::unique_ptr<CallInst>(new CallInst(I, *this));
+  return cast<CallInst>(registerValue(std::move(NewPtr)));
+}
+
+InvokeInst *Context::createInvokeInst(llvm::InvokeInst *I) {
+  auto NewPtr = std::unique_ptr<InvokeInst>(new InvokeInst(I, *this));
+  return cast<InvokeInst>(registerValue(std::move(NewPtr)));
+}
+
+CallBrInst *Context::createCallBrInst(llvm::CallBrInst *I) {
+  auto NewPtr = std::unique_ptr<CallBrInst>(new CallBrInst(I, *this));
+  return cast<CallBrInst>(registerValue(std::move(NewPtr)));
+}
+
+UnreachableInst *Context::createUnreachableInst(llvm::UnreachableInst *UI) {
+  auto NewPtr =
+      std::unique_ptr<UnreachableInst>(new UnreachableInst(UI, *this));
+  return cast<UnreachableInst>(registerValue(std::move(NewPtr)));
+}
+LandingPadInst *Context::createLandingPadInst(llvm::LandingPadInst *I) {
+  auto NewPtr = std::unique_ptr<LandingPadInst>(new LandingPadInst(I, *this));
+  return cast<LandingPadInst>(registerValue(std::move(NewPtr)));
+}
+CatchPadInst *Context::createCatchPadInst(llvm::CatchPadInst *I) {
+  auto NewPtr = std::unique_ptr<CatchPadInst>(new CatchPadInst(I, *this));
+  return cast<CatchPadInst>(registerValue(std::move(NewPtr)));
+}
+CleanupPadInst *Context::createCleanupPadInst(llvm::CleanupPadInst *I) {
+  auto NewPtr = std::unique_ptr<CleanupPadInst>(new CleanupPadInst(I, *this));
+  return cast<CleanupPadInst>(registerValue(std::move(NewPtr)));
+}
+CatchReturnInst *Context::createCatchReturnInst(llvm::CatchReturnInst *I) {
+  auto NewPtr = std::unique_ptr<CatchReturnInst>(new CatchReturnInst(I, *this));
+  return cast<CatchReturnInst>(registerValue(std::move(NewPtr)));
+}
+CleanupReturnInst *
+Context::createCleanupReturnInst(llvm::CleanupReturnInst *I) {
+  auto NewPtr =
+      std::unique_ptr<CleanupReturnInst>(new CleanupReturnInst(I, *this));
+  return cast<CleanupReturnInst>(registerValue(std::move(NewPtr)));
+}
+GetElementPtrInst *
+Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
+  auto NewPtr =
+      std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
+  return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
+}
+CatchSwitchInst *Context::createCatchSwitchInst(llvm::CatchSwitchInst *I) {
+  auto NewPtr = std::unique_ptr<CatchSwitchInst>(new CatchSwitchInst(I, *this));
+  return cast<CatchSwitchInst>(registerValue(std::move(NewPtr)));
+}
+ResumeInst *Context::createResumeInst(llvm::ResumeInst *I) {
+  auto NewPtr = std::unique_ptr<ResumeInst>(new ResumeInst(I, *this));
+  return cast<ResumeInst>(registerValue(std::move(NewPtr)));
+}
+SwitchInst *Context::createSwitchInst(llvm::SwitchInst *I) {
+  auto NewPtr = std::unique_ptr<SwitchInst>(new SwitchInst(I, *this));
+  return cast<SwitchInst>(registerValue(std::move(NewPtr)));
+}
+UnaryOperator *Context::createUnaryOperator(llvm::UnaryOperator *I) {
+  auto NewPtr = std::unique_ptr<UnaryOperator>(new UnaryOperator(I, *this));
+  return cast<UnaryOperator>(registerValue(std::move(NewPtr)));
+}
+BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
+  auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
+  return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
+}
+AtomicRMWInst *Context::createAtomicRMWInst(llvm::AtomicRMWInst *I) {
+  auto NewPtr = std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(I, *this));
+  return cast<AtomicRMWInst>(registerValue(std::move(NewPtr)));
+}
+AtomicCmpXchgInst *
+Context::createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I) {
+  auto NewPtr =
+      std::unique_ptr<AtomicCmpXchgInst>(new AtomicCmpXchgInst(I, *this));
+  return cast<AtomicCmpXchgInst>(registerValue(std::move(NewPtr)));
+}
+AllocaInst *Context::createAllocaInst(llvm::AllocaInst *I) {
+  auto NewPtr = std::unique_ptr<AllocaInst>(new AllocaInst(I, *this));
+  return cast<AllocaInst>(registerValue(std::move(NewPtr)));
+}
+CastInst *Context::createCastInst(llvm::CastInst *I) {
+  auto NewPtr = std::unique_ptr<CastInst>(new CastInst(I, *this));
+  return cast<CastInst>(registerValue(std::move(NewPtr)));
+}
+PHINode *Context::createPHINode(llvm::PHINode *I) {
+  auto NewPtr = std::unique_ptr<PHINode>(new PHINode(I, *this));
+  return cast<PHINode>(registerValue(std::move(NewPtr)));
+}
+ICmpInst *Context::createICmpInst(llvm::ICmpInst *I) {
+  auto NewPtr = std::unique_ptr<ICmpInst>(new ICmpInst(I, *this));
+  return cast<ICmpInst>(registerValue(std::move(NewPtr)));
+}
+FCmpInst *Context::createFCmpInst(llvm::FCmpInst *I) {
+  auto NewPtr = std::unique_ptr<FCmpInst>(new FCmpInst(I, *this));
+  return cast<FCmpInst>(registerValue(std::move(NewPtr)));
+}
+CmpInst *CmpInst::create(Predicate P, Value *S1, Value *S2,
+                         Instruction *InsertBefore, Context &Ctx,
+                         const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  auto *LLVMI = Builder.CreateCmp(P, S1->Val, S2->Val, Name);
+  if (dyn_cast<llvm::ICmpInst>(LLVMI))
+    return Ctx.createICmpInst(cast<llvm::ICmpInst>(LLVMI));
+  return Ctx.createFCmpInst(cast<llvm::FCmpInst>(LLVMI));
+}
+CmpInst *CmpInst::createWithCopiedFlags(Predicate P, Value *S1, Value *S2,
+                                        const Instruction *F,
+                                        Instruction *InsertBefore, Context &Ctx,
+                                        const Twine &Name) {
+  CmpInst *Inst = create(P, S1, S2, InsertBefore, Ctx, Name);
+  cast<llvm::CmpInst>(Inst->Val)->copyIRFlags(F->Val);
+  return Inst;
+}
+
+Type *CmpInst::makeCmpResultType(Type *OpndType) {
+  if (auto *VT = dyn_cast<VectorType>(OpndType)) {
+    // TODO: Cleanup when we have more complete support for
+    // sandboxir::VectorType
+    return OpndType->getContext().getType(llvm::VectorType::get(
+        llvm::Type::getInt1Ty(OpndType->getContext().LLVMCtx),
+        cast<llvm::VectorType>(VT->LLVMTy)->getElementCount()));
+  }
+  return Type::getInt1Ty(OpndType->getContext());
+}
+
+void CmpInst::setPredicate(Predicate P) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&CmpInst::getPredicate, &CmpInst::setPredicate>>(this);
+  cast<llvm::CmpInst>(Val)->setPredicate(P);
+}
+
+void CmpInst::swapOperands() {
+  if (ICmpInst *IC = dyn_cast<ICmpInst>(this))
+    IC->swapOperands();
+  else
+    cast<FCmpInst>(this)->swapOperands();
+}
+
+void ICmpInst::swapOperands() {
+  Ctx.getTracker().emplaceIfTracking<CmpSwapOperands>(this);
+  cast<llvm::ICmpInst>(Val)->swapOperands();
+}
+
+void FCmpInst::swapOperands() {
+  Ctx.getTracker().emplaceIfTracking<CmpSwapOperands>(this);
+  cast<llvm::FCmpInst>(Val)->swapOperands();
+}
+
+#ifndef NDEBUG
+void CmpInst::dumpOS(raw_ostream &OS) const {
+  dumpCommonPrefix(OS);
+  dumpCommonSuffix(OS);
+}
+
+void CmpInst::dump() const {
+  dumpOS(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
+Value *Context::getValue(llvm::Value *V) const {
+  auto It = LLVMValueToValueMap.find(V);
+  if (It != LLVMValueToValueMap.end())
+    return It->second.get();
+  return nullptr;
+}
+
+Context::Context(LLVMContext &LLVMCtx)
+    : LLVMCtx(LLVMCtx), IRTracker(*this),
+      LLVMIRBuilder(LLVMCtx, ConstantFolder()) {}
+
+Context::~Context() {}
+
+Module *Context::getModule(llvm::Module *LLVMM) const {
+  auto It = LLVMModuleToModuleMap.find(LLVMM);
+  if (It != LLVMModuleToModuleMap.end())
+    return It->second.get();
+  return nullptr;
+}
+
+Module *Context::getOrCreateModule(llvm::Module *LLVMM) {
+  auto Pair = LLVMModuleToModuleMap.insert({LLVMM, nullptr});
+  auto It = Pair.first;
+  if (!Pair.second)
+    return It->second.get();
+  It->second = std::unique_ptr<Module>(new Module(*LLVMM, *this));
+  return It->second.get();
+}
+
+Function *Context::createFunction(llvm::Function *F) {
+  assert(getValue(F) == nullptr && "Already exists!");
+  // Create the module if needed before we create the new sandboxir::Function.
+  // Note: this won't fully populate the module. The only globals that will be
+  // available will be the ones being used within the function.
+  getOrCreateModule(F->getParent());
+
+  auto NewFPtr = std::unique_ptr<Function>(new Function(F, *this));
+  auto *SBF = cast<Function>(registerValue(std::move(NewFPtr)));
+  // Create arguments.
+  for (auto &Arg : F->args())
+    getOrCreateArgument(&Arg);
+  // Create BBs.
+  for (auto &BB : *F)
+    createBasicBlock(&BB);
+  return SBF;
+}
+
+Module *Context::createModule(llvm::Module *LLVMM) {
+  auto *M = getOrCreateModule(LLVMM);
+  // Create the functions.
+  for (auto &LLVMF : *LLVMM)
+    createFunction(&LLVMF);
+  // Create globals.
+  for (auto &Global : LLVMM->globals())
+    getOrCreateValue(&Global);
+  // Create aliases.
+  for (auto &Alias : LLVMM->aliases())
+    getOrCreateValue(&Alias);
+  // Create ifuncs.
+  for (auto &IFunc : LLVMM->ifuncs())
+    getOrCreateValue(&IFunc);
+
+  return M;
+}
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Instruction.cpp b/llvm/lib/SandboxIR/Instruction.cpp
new file mode 100644
index 0000000000000..919a44fca8b04
--- /dev/null
+++ b/llvm/lib/SandboxIR/Instruction.cpp
@@ -0,0 +1,1965 @@
+//===- Instruction.cpp - The Instructions of Sandbox IR -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Instruction.h"
+
+namespace llvm::sandboxir {
+
+const char *Instruction::getOpcodeName(Opcode Opc) {
+  switch (Opc) {
+#define OP(OPC)                                                                \
+  case Opcode::OPC:                                                            \
+    return #OPC;
+#define OPCODES(...) __VA_ARGS__
+#define DEF_INSTR(ID, OPC, CLASS) OPC
+#include "llvm/SandboxIR/SandboxIRValues.def"
+  }
+  llvm_unreachable("Unknown Opcode");
+}
+
+llvm::Instruction *Instruction::getTopmostLLVMInstruction() const {
+  Instruction *Prev = getPrevNode();
+  if (Prev == nullptr) {
+    // If at top of the BB, return the first BB instruction.
+    return &*cast<llvm::BasicBlock>(getParent()->Val)->begin();
+  }
+  // Else get the Previous sandbox IR instruction's bottom IR instruction and
+  // return its successor.
+  llvm::Instruction *PrevBotI = cast<llvm::Instruction>(Prev->Val);
+  return PrevBotI->getNextNode();
+}
+
+BBIterator Instruction::getIterator() const {
+  auto *I = cast<llvm::Instruction>(Val);
+  return BasicBlock::iterator(I->getParent(), I->getIterator(), &Ctx);
+}
+
+Instruction *Instruction::getNextNode() const {
+  assert(getParent() != nullptr && "Detached!");
+  assert(getIterator() != getParent()->end() && "Already at end!");
+  // `Val` is the bottom-most LLVM IR instruction. Get the next in the chain,
+  // and get the corresponding sandboxir Instruction that maps to it. This works
+  // even for SandboxIR Instructions that map to more than one LLVM Instruction.
+  auto *LLVMI = cast<llvm::Instruction>(Val);
+  assert(LLVMI->getParent() != nullptr && "LLVM IR instr is detached!");
+  auto *NextLLVMI = LLVMI->getNextNode();
+  auto *NextI = cast_or_null<Instruction>(Ctx.getValue(NextLLVMI));
+  if (NextI == nullptr)
+    return nullptr;
+  return NextI;
+}
+
+Instruction *Instruction::getPrevNode() const {
+  assert(getParent() != nullptr && "Detached!");
+  auto It = getIterator();
+  if (It != getParent()->begin())
+    return std::prev(getIterator()).get();
+  return nullptr;
+}
+
+void Instruction::removeFromParent() {
+  Ctx.getTracker().emplaceIfTracking<RemoveFromParent>(this);
+
+  // Detach all the LLVM IR instructions from their parent BB.
+  for (llvm::Instruction *I : getLLVMInstrs())
+    I->removeFromParent();
+}
+
+void Instruction::eraseFromParent() {
+  assert(users().empty() && "Still connected to users, can't erase!");
+  std::unique_ptr<Value> Detached = Ctx.detach(this);
+  auto LLVMInstrs = getLLVMInstrs();
+
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking()) {
+    Tracker.track(std::make_unique<EraseFromParent>(std::move(Detached)));
+    // We don't actually delete the IR instruction, because then it would be
+    // impossible to bring it back from the dead at the same memory location.
+    // Instead we remove it from its BB and track its current location.
+    for (llvm::Instruction *I : LLVMInstrs)
+      I->removeFromParent();
+    // TODO: Multi-instructions need special treatment because some of the
+    // references are internal to the instruction.
+    for (llvm::Instruction *I : LLVMInstrs)
+      I->dropAllReferences();
+  } else {
+    // Erase in reverse to avoid erasing nstructions with attached uses.
+    for (llvm::Instruction *I : reverse(LLVMInstrs))
+      I->eraseFromParent();
+  }
+}
+
+void Instruction::moveBefore(BasicBlock &BB, const BBIterator &WhereIt) {
+  if (std::next(getIterator()) == WhereIt)
+    // Destination is same as origin, nothing to do.
+    return;
+
+  Ctx.getTracker().emplaceIfTracking<MoveInstr>(this);
+
+  auto *LLVMBB = cast<llvm::BasicBlock>(BB.Val);
+  llvm::BasicBlock::iterator It;
+  if (WhereIt == BB.end()) {
+    It = LLVMBB->end();
+  } else {
+    Instruction *WhereI = &*WhereIt;
+    It = WhereI->getTopmostLLVMInstruction()->getIterator();
+  }
+  // TODO: Move this to the verifier of sandboxir::Instruction.
+  assert(is_sorted(getLLVMInstrs(),
+                   [](auto *I1, auto *I2) { return I1->comesBefore(I2); }) &&
+         "Expected program order!");
+  // Do the actual move in LLVM IR.
+  for (auto *I : getLLVMInstrs())
+    I->moveBefore(*LLVMBB, It);
+}
+
+void Instruction::insertBefore(Instruction *BeforeI) {
+  llvm::Instruction *BeforeTopI = BeforeI->getTopmostLLVMInstruction();
+  // TODO: Move this to the verifier of sandboxir::Instruction.
+  assert(is_sorted(getLLVMInstrs(),
+                   [](auto *I1, auto *I2) { return I1->comesBefore(I2); }) &&
+         "Expected program order!");
+
+  Ctx.getTracker().emplaceIfTracking<InsertIntoBB>(this);
+
+  // Insert the LLVM IR Instructions in program order.
+  for (llvm::Instruction *I : getLLVMInstrs())
+    I->insertBefore(BeforeTopI);
+}
+
+void Instruction::insertAfter(Instruction *AfterI) {
+  insertInto(AfterI->getParent(), std::next(AfterI->getIterator()));
+}
+
+void Instruction::insertInto(BasicBlock *BB, const BBIterator &WhereIt) {
+  llvm::BasicBlock *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
+  llvm::Instruction *LLVMBeforeI;
+  llvm::BasicBlock::iterator LLVMBeforeIt;
+  Instruction *BeforeI;
+  if (WhereIt != BB->end()) {
+    BeforeI = &*WhereIt;
+    LLVMBeforeI = BeforeI->getTopmostLLVMInstruction();
+    LLVMBeforeIt = LLVMBeforeI->getIterator();
+  } else {
+    BeforeI = nullptr;
+    LLVMBeforeI = nullptr;
+    LLVMBeforeIt = LLVMBB->end();
+  }
+
+  Ctx.getTracker().emplaceIfTracking<InsertIntoBB>(this);
+
+  // Insert the LLVM IR Instructions in program order.
+  for (llvm::Instruction *I : getLLVMInstrs())
+    I->insertInto(LLVMBB, LLVMBeforeIt);
+}
+
+BasicBlock *Instruction::getParent() const {
+  // Get the LLVM IR Instruction that this maps to, get its parent, and get the
+  // corresponding sandboxir::BasicBlock by looking it up in sandboxir::Context.
+  auto *BB = cast<llvm::Instruction>(Val)->getParent();
+  if (BB == nullptr)
+    return nullptr;
+  return cast<BasicBlock>(Ctx.getValue(BB));
+}
+
+bool Instruction::classof(const sandboxir::Value *From) {
+  switch (From->getSubclassID()) {
+#define DEF_INSTR(ID, OPC, CLASS)                                              \
+  case ClassID::ID:                                                            \
+    return true;
+#include "llvm/SandboxIR/SandboxIRValues.def"
+  default:
+    return false;
+  }
+}
+
+void Instruction::setHasNoUnsignedWrap(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasNoUnsignedWrap,
+                                       &Instruction::setHasNoUnsignedWrap>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasNoUnsignedWrap(B);
+}
+
+void Instruction::setHasNoSignedWrap(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasNoSignedWrap,
+                                       &Instruction::setHasNoSignedWrap>>(this);
+  cast<llvm::Instruction>(Val)->setHasNoSignedWrap(B);
+}
+
+void Instruction::setFast(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&Instruction::isFast, &Instruction::setFast>>(this);
+  cast<llvm::Instruction>(Val)->setFast(B);
+}
+
+void Instruction::setIsExact(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&Instruction::isExact, &Instruction::setIsExact>>(this);
+  cast<llvm::Instruction>(Val)->setIsExact(B);
+}
+
+void Instruction::setHasAllowReassoc(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowReassoc,
+                                       &Instruction::setHasAllowReassoc>>(this);
+  cast<llvm::Instruction>(Val)->setHasAllowReassoc(B);
+}
+
+void Instruction::setHasNoNaNs(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&Instruction::hasNoNaNs, &Instruction::setHasNoNaNs>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasNoNaNs(B);
+}
+
+void Instruction::setHasNoInfs(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&Instruction::hasNoInfs, &Instruction::setHasNoInfs>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasNoInfs(B);
+}
+
+void Instruction::setHasNoSignedZeros(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasNoSignedZeros,
+                                       &Instruction::setHasNoSignedZeros>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasNoSignedZeros(B);
+}
+
+void Instruction::setHasAllowReciprocal(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowReciprocal,
+                                       &Instruction::setHasAllowReciprocal>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasAllowReciprocal(B);
+}
+
+void Instruction::setHasAllowContract(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowContract,
+                                       &Instruction::setHasAllowContract>>(
+          this);
+  cast<llvm::Instruction>(Val)->setHasAllowContract(B);
+}
+
+void Instruction::setFastMathFlags(FastMathFlags FMF) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::getFastMathFlags,
+                                       &Instruction::copyFastMathFlags>>(this);
+  cast<llvm::Instruction>(Val)->setFastMathFlags(FMF);
+}
+
+void Instruction::copyFastMathFlags(FastMathFlags FMF) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::getFastMathFlags,
+                                       &Instruction::copyFastMathFlags>>(this);
+  cast<llvm::Instruction>(Val)->copyFastMathFlags(FMF);
+}
+
+Type *Instruction::getAccessType() const {
+  return Ctx.getType(cast<llvm::Instruction>(Val)->getAccessType());
+}
+
+void Instruction::setHasApproxFunc(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&Instruction::hasApproxFunc,
+                                       &Instruction::setHasApproxFunc>>(this);
+  cast<llvm::Instruction>(Val)->setHasApproxFunc(B);
+}
+
+#ifndef NDEBUG
+void Instruction::dumpOS(raw_ostream &OS) const {
+  OS << "Unimplemented! Please override dump().";
+}
+#endif // NDEBUG
+
+VAArgInst *VAArgInst::create(Value *List, Type *Ty, BBIterator WhereIt,
+                             BasicBlock *WhereBB, Context &Ctx,
+                             const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  auto *LLVMI =
+      cast<llvm::VAArgInst>(Builder.CreateVAArg(List->Val, Ty->LLVMTy, Name));
+  return Ctx.createVAArgInst(LLVMI);
+}
+
+Value *VAArgInst::getPointerOperand() {
+  return Ctx.getValue(cast<llvm::VAArgInst>(Val)->getPointerOperand());
+}
+
+FreezeInst *FreezeInst::create(Value *V, BBIterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  auto *LLVMI = cast<llvm::FreezeInst>(Builder.CreateFreeze(V->Val, Name));
+  return Ctx.createFreezeInst(LLVMI);
+}
+
+FenceInst *FenceInst::create(AtomicOrdering Ordering, BBIterator WhereIt,
+                             BasicBlock *WhereBB, Context &Ctx,
+                             SyncScope::ID SSID) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::FenceInst *LLVMI = Builder.CreateFence(Ordering, SSID);
+  return Ctx.createFenceInst(LLVMI);
+}
+
+void FenceInst::setOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&FenceInst::getOrdering, &FenceInst::setOrdering>>(
+          this);
+  cast<llvm::FenceInst>(Val)->setOrdering(Ordering);
+}
+
+void FenceInst::setSyncScopeID(SyncScope::ID SSID) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&FenceInst::getSyncScopeID,
+                                       &FenceInst::setSyncScopeID>>(this);
+  cast<llvm::FenceInst>(Val)->setSyncScopeID(SSID);
+}
+
+Value *SelectInst::createCommon(Value *Cond, Value *True, Value *False,
+                                const Twine &Name, IRBuilder<> &Builder,
+                                Context &Ctx) {
+  llvm::Value *NewV =
+      Builder.CreateSelect(Cond->Val, True->Val, False->Val, Name);
+  if (auto *NewSI = dyn_cast<llvm::SelectInst>(NewV))
+    return Ctx.createSelectInst(NewSI);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *SelectInst::create(Value *Cond, Value *True, Value *False,
+                          Instruction *InsertBefore, Context &Ctx,
+                          const Twine &Name) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  return createCommon(Cond, True, False, Name, Builder, Ctx);
+}
+
+Value *SelectInst::create(Value *Cond, Value *True, Value *False,
+                          BasicBlock *InsertAtEnd, Context &Ctx,
+                          const Twine &Name) {
+  auto *IRInsertAtEnd = cast<llvm::BasicBlock>(InsertAtEnd->Val);
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(IRInsertAtEnd);
+  return createCommon(Cond, True, False, Name, Builder, Ctx);
+}
+
+void SelectInst::swapValues() {
+  Ctx.getTracker().emplaceIfTracking<UseSwap>(getOperandUse(1),
+                                              getOperandUse(2));
+  cast<llvm::SelectInst>(Val)->swapValues();
+}
+
+bool SelectInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Select;
+}
+
+BranchInst *BranchInst::create(BasicBlock *IfTrue, Instruction *InsertBefore,
+                               Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
+  Builder.SetInsertPoint(cast<llvm::Instruction>(LLVMBefore));
+  llvm::BranchInst *NewBr =
+      Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
+  return Ctx.createBranchInst(NewBr);
+}
+
+BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd,
+                               Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::BranchInst *NewBr =
+      Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
+  return Ctx.createBranchInst(NewBr);
+}
+
+BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
+                               Value *Cond, Instruction *InsertBefore,
+                               Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
+  Builder.SetInsertPoint(LLVMBefore);
+  llvm::BranchInst *NewBr =
+      Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
+                           cast<llvm::BasicBlock>(IfFalse->Val));
+  return Ctx.createBranchInst(NewBr);
+}
+
+BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
+                               Value *Cond, BasicBlock *InsertAtEnd,
+                               Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::BranchInst *NewBr =
+      Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
+                           cast<llvm::BasicBlock>(IfFalse->Val));
+  return Ctx.createBranchInst(NewBr);
+}
+
+bool BranchInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Br;
+}
+
+Value *BranchInst::getCondition() const {
+  assert(isConditional() && "Cannot get condition of an uncond branch!");
+  return Ctx.getValue(cast<llvm::BranchInst>(Val)->getCondition());
+}
+
+BasicBlock *BranchInst::getSuccessor(unsigned SuccIdx) const {
+  assert(SuccIdx < getNumSuccessors() &&
+         "Successor # out of range for Branch!");
+  return cast_or_null<BasicBlock>(
+      Ctx.getValue(cast<llvm::BranchInst>(Val)->getSuccessor(SuccIdx)));
+}
+
+void BranchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
+  assert((Idx == 0 || Idx == 1) && "Out of bounds!");
+  setOperand(2u - Idx, NewSucc);
+}
+
+BasicBlock *BranchInst::LLVMBBToSBBB::operator()(llvm::BasicBlock *BB) const {
+  return cast<BasicBlock>(Ctx.getValue(BB));
+}
+const BasicBlock *
+BranchInst::ConstLLVMBBToSBBB::operator()(const llvm::BasicBlock *BB) const {
+  return cast<BasicBlock>(Ctx.getValue(BB));
+}
+
+void LoadInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&LoadInst::isVolatile, &LoadInst::setVolatile>>(this);
+  cast<llvm::LoadInst>(Val)->setVolatile(V);
+}
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           Instruction *InsertBefore, Context &Ctx,
+                           const Twine &Name) {
+  return create(Ty, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx, Name);
+}
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           Instruction *InsertBefore, bool IsVolatile,
+                           Context &Ctx, const Twine &Name) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  auto *NewLI =
+      Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
+  auto *NewSBI = Ctx.createLoadInst(NewLI);
+  return NewSBI;
+}
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           BasicBlock *InsertAtEnd, Context &Ctx,
+                           const Twine &Name) {
+  return create(Ty, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx, Name);
+}
+
+LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
+                           BasicBlock *InsertAtEnd, bool IsVolatile,
+                           Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  auto *NewLI =
+      Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
+  auto *NewSBI = Ctx.createLoadInst(NewLI);
+  return NewSBI;
+}
+
+bool LoadInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Load;
+}
+
+Value *LoadInst::getPointerOperand() const {
+  return Ctx.getValue(cast<llvm::LoadInst>(Val)->getPointerOperand());
+}
+
+void StoreInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&StoreInst::isVolatile, &StoreInst::setVolatile>>(this);
+  cast<llvm::StoreInst>(Val)->setVolatile(V);
+}
+
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             Instruction *InsertBefore, Context &Ctx) {
+  return create(V, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx);
+}
+
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             Instruction *InsertBefore, bool IsVolatile,
+                             Context &Ctx) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
+  auto *NewSBI = Ctx.createStoreInst(NewSI);
+  return NewSBI;
+}
+
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             BasicBlock *InsertAtEnd, Context &Ctx) {
+  return create(V, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx);
+}
+
+StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
+                             BasicBlock *InsertAtEnd, bool IsVolatile,
+                             Context &Ctx) {
+  auto *InsertAtEndIR = cast<llvm::BasicBlock>(InsertAtEnd->Val);
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertAtEndIR);
+  auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
+  auto *NewSBI = Ctx.createStoreInst(NewSI);
+  return NewSBI;
+}
+
+bool StoreInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Store;
+}
+
+Value *StoreInst::getValueOperand() const {
+  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getValueOperand());
+}
+
+Value *StoreInst::getPointerOperand() const {
+  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getPointerOperand());
+}
+
+UnreachableInst *UnreachableInst::create(Instruction *InsertBefore,
+                                         Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
+  Builder.SetInsertPoint(LLVMBefore);
+  llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
+  return Ctx.createUnreachableInst(NewUI);
+}
+
+UnreachableInst *UnreachableInst::create(BasicBlock *InsertAtEnd,
+                                         Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
+  return Ctx.createUnreachableInst(NewUI);
+}
+
+bool UnreachableInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Unreachable;
+}
+
+ReturnInst *ReturnInst::createCommon(Value *RetVal, IRBuilder<> &Builder,
+                                     Context &Ctx) {
+  llvm::ReturnInst *NewRI;
+  if (RetVal != nullptr)
+    NewRI = Builder.CreateRet(RetVal->Val);
+  else
+    NewRI = Builder.CreateRetVoid();
+  return Ctx.createReturnInst(NewRI);
+}
+
+ReturnInst *ReturnInst::create(Value *RetVal, Instruction *InsertBefore,
+                               Context &Ctx) {
+  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(BeforeIR);
+  return createCommon(RetVal, Builder, Ctx);
+}
+
+ReturnInst *ReturnInst::create(Value *RetVal, BasicBlock *InsertAtEnd,
+                               Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  return createCommon(RetVal, Builder, Ctx);
+}
+
+Value *ReturnInst::getReturnValue() const {
+  auto *LLVMRetVal = cast<llvm::ReturnInst>(Val)->getReturnValue();
+  return LLVMRetVal != nullptr ? Ctx.getValue(LLVMRetVal) : nullptr;
+}
+
+FunctionType *CallBase::getFunctionType() const {
+  return cast<FunctionType>(
+      Ctx.getType(cast<llvm::CallBase>(Val)->getFunctionType()));
+}
+
+Value *CallBase::getCalledOperand() const {
+  return Ctx.getValue(cast<llvm::CallBase>(Val)->getCalledOperand());
+}
+
+Use CallBase::getCalledOperandUse() const {
+  llvm::Use *LLVMUse = &cast<llvm::CallBase>(Val)->getCalledOperandUse();
+  return Use(LLVMUse, cast<User>(Ctx.getValue(LLVMUse->getUser())), Ctx);
+}
+
+Function *CallBase::getCalledFunction() const {
+  return cast_or_null<Function>(
+      Ctx.getValue(cast<llvm::CallBase>(Val)->getCalledFunction()));
+}
+Function *CallBase::getCaller() {
+  return cast<Function>(Ctx.getValue(cast<llvm::CallBase>(Val)->getCaller()));
+}
+
+void CallBase::setCalledFunction(Function *F) {
+  // F's function type is private, so we rely on `setCalledFunction()` to update
+  // it. But even though we are calling `setCalledFunction()` we also need to
+  // track this change at the SandboxIR level, which is why we call
+  // `setCalledOperand()` here.
+  // Note: This may break if `setCalledFunction()` early returns if `F`
+  // is already set, but we do have a unit test for it.
+  setCalledOperand(F);
+  cast<llvm::CallBase>(Val)->setCalledFunction(
+      cast<llvm::FunctionType>(F->getFunctionType()->LLVMTy),
+      cast<llvm::Function>(F->Val));
+}
+
+CallInst *CallInst::create(FunctionType *FTy, Value *Func,
+                           ArrayRef<Value *> Args, BasicBlock::iterator WhereIt,
+                           BasicBlock *WhereBB, Context &Ctx,
+                           const Twine &NameStr) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  SmallVector<llvm::Value *> LLVMArgs;
+  LLVMArgs.reserve(Args.size());
+  for (Value *Arg : Args)
+    LLVMArgs.push_back(Arg->Val);
+  llvm::CallInst *NewCI = Builder.CreateCall(
+      cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val, LLVMArgs, NameStr);
+  return Ctx.createCallInst(NewCI);
+}
+
+CallInst *CallInst::create(FunctionType *FTy, Value *Func,
+                           ArrayRef<Value *> Args, Instruction *InsertBefore,
+                           Context &Ctx, const Twine &NameStr) {
+  return CallInst::create(FTy, Func, Args, InsertBefore->getIterator(),
+                          InsertBefore->getParent(), Ctx, NameStr);
+}
+
+CallInst *CallInst::create(FunctionType *FTy, Value *Func,
+                           ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                           Context &Ctx, const Twine &NameStr) {
+  return CallInst::create(FTy, Func, Args, InsertAtEnd->end(), InsertAtEnd, Ctx,
+                          NameStr);
+}
+
+InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *IfNormal, BasicBlock *IfException,
+                               ArrayRef<Value *> Args, BBIterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &NameStr) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  SmallVector<llvm::Value *> LLVMArgs;
+  LLVMArgs.reserve(Args.size());
+  for (Value *Arg : Args)
+    LLVMArgs.push_back(Arg->Val);
+  llvm::InvokeInst *Invoke = Builder.CreateInvoke(
+      cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val,
+      cast<llvm::BasicBlock>(IfNormal->Val),
+      cast<llvm::BasicBlock>(IfException->Val), LLVMArgs, NameStr);
+  return Ctx.createInvokeInst(Invoke);
+}
+
+InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *IfNormal, BasicBlock *IfException,
+                               ArrayRef<Value *> Args,
+                               Instruction *InsertBefore, Context &Ctx,
+                               const Twine &NameStr) {
+  return create(FTy, Func, IfNormal, IfException, Args,
+                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
+                NameStr);
+}
+
+InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *IfNormal, BasicBlock *IfException,
+                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                               Context &Ctx, const Twine &NameStr) {
+  return create(FTy, Func, IfNormal, IfException, Args, InsertAtEnd->end(),
+                InsertAtEnd, Ctx, NameStr);
+}
+
+BasicBlock *InvokeInst::getNormalDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getNormalDest()));
+}
+BasicBlock *InvokeInst::getUnwindDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getUnwindDest()));
+}
+void InvokeInst::setNormalDest(BasicBlock *BB) {
+  setOperand(1, BB);
+  assert(getNormalDest() == BB && "LLVM IR uses a different operan index!");
+}
+void InvokeInst::setUnwindDest(BasicBlock *BB) {
+  setOperand(2, BB);
+  assert(getUnwindDest() == BB && "LLVM IR uses a different operan index!");
+}
+LandingPadInst *InvokeInst::getLandingPadInst() const {
+  return cast<LandingPadInst>(
+      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getLandingPadInst()));
+  ;
+}
+BasicBlock *InvokeInst::getSuccessor(unsigned SuccIdx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getSuccessor(SuccIdx)));
+}
+
+CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *DefaultDest,
+                               ArrayRef<BasicBlock *> IndirectDests,
+                               ArrayRef<Value *> Args, BBIterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &NameStr) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+
+  SmallVector<llvm::BasicBlock *> LLVMIndirectDests;
+  LLVMIndirectDests.reserve(IndirectDests.size());
+  for (BasicBlock *IndDest : IndirectDests)
+    LLVMIndirectDests.push_back(cast<llvm::BasicBlock>(IndDest->Val));
+
+  SmallVector<llvm::Value *> LLVMArgs;
+  LLVMArgs.reserve(Args.size());
+  for (Value *Arg : Args)
+    LLVMArgs.push_back(Arg->Val);
+
+  llvm::CallBrInst *CallBr =
+      Builder.CreateCallBr(cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val,
+                           cast<llvm::BasicBlock>(DefaultDest->Val),
+                           LLVMIndirectDests, LLVMArgs, NameStr);
+  return Ctx.createCallBrInst(CallBr);
+}
+
+CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *DefaultDest,
+                               ArrayRef<BasicBlock *> IndirectDests,
+                               ArrayRef<Value *> Args,
+                               Instruction *InsertBefore, Context &Ctx,
+                               const Twine &NameStr) {
+  return create(FTy, Func, DefaultDest, IndirectDests, Args,
+                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
+                NameStr);
+}
+CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
+                               BasicBlock *DefaultDest,
+                               ArrayRef<BasicBlock *> IndirectDests,
+                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
+                               Context &Ctx, const Twine &NameStr) {
+  return create(FTy, Func, DefaultDest, IndirectDests, Args, InsertAtEnd->end(),
+                InsertAtEnd, Ctx, NameStr);
+}
+
+Value *CallBrInst::getIndirectDestLabel(unsigned Idx) const {
+  return Ctx.getValue(cast<llvm::CallBrInst>(Val)->getIndirectDestLabel(Idx));
+}
+Value *CallBrInst::getIndirectDestLabelUse(unsigned Idx) const {
+  return Ctx.getValue(
+      cast<llvm::CallBrInst>(Val)->getIndirectDestLabelUse(Idx));
+}
+BasicBlock *CallBrInst::getDefaultDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getDefaultDest()));
+}
+BasicBlock *CallBrInst::getIndirectDest(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getIndirectDest(Idx)));
+}
+llvm::SmallVector<BasicBlock *, 16> CallBrInst::getIndirectDests() const {
+  SmallVector<BasicBlock *, 16> BBs;
+  for (llvm::BasicBlock *LLVMBB :
+       cast<llvm::CallBrInst>(Val)->getIndirectDests())
+    BBs.push_back(cast<BasicBlock>(Ctx.getValue(LLVMBB)));
+  return BBs;
+}
+void CallBrInst::setDefaultDest(BasicBlock *BB) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CallBrInst::getDefaultDest,
+                                       &CallBrInst::setDefaultDest>>(this);
+  cast<llvm::CallBrInst>(Val)->setDefaultDest(cast<llvm::BasicBlock>(BB->Val));
+}
+void CallBrInst::setIndirectDest(unsigned Idx, BasicBlock *BB) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&CallBrInst::getIndirectDest,
+                                              &CallBrInst::setIndirectDest>>(
+          this, Idx);
+  cast<llvm::CallBrInst>(Val)->setIndirectDest(Idx,
+                                               cast<llvm::BasicBlock>(BB->Val));
+}
+BasicBlock *CallBrInst::getSuccessor(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getSuccessor(Idx)));
+}
+
+LandingPadInst *LandingPadInst::create(Type *RetTy, unsigned NumReservedClauses,
+                                       BBIterator WhereIt, BasicBlock *WhereBB,
+                                       Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::LandingPadInst *LLVMI =
+      Builder.CreateLandingPad(RetTy->LLVMTy, NumReservedClauses, Name);
+  return Ctx.createLandingPadInst(LLVMI);
+}
+
+void LandingPadInst::setCleanup(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&LandingPadInst::isCleanup,
+                                       &LandingPadInst::setCleanup>>(this);
+  cast<llvm::LandingPadInst>(Val)->setCleanup(V);
+}
+
+Constant *LandingPadInst::getClause(unsigned Idx) const {
+  return cast<Constant>(
+      Ctx.getValue(cast<llvm::LandingPadInst>(Val)->getClause(Idx)));
+}
+
+Value *FuncletPadInst::getParentPad() const {
+  return Ctx.getValue(cast<llvm::FuncletPadInst>(Val)->getParentPad());
+}
+
+void FuncletPadInst::setParentPad(Value *ParentPad) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&FuncletPadInst::getParentPad,
+                                       &FuncletPadInst::setParentPad>>(this);
+  cast<llvm::FuncletPadInst>(Val)->setParentPad(ParentPad->Val);
+}
+
+Value *FuncletPadInst::getArgOperand(unsigned Idx) const {
+  return Ctx.getValue(cast<llvm::FuncletPadInst>(Val)->getArgOperand(Idx));
+}
+
+void FuncletPadInst::setArgOperand(unsigned Idx, Value *V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&FuncletPadInst::getArgOperand,
+                                              &FuncletPadInst::setArgOperand>>(
+          this, Idx);
+  cast<llvm::FuncletPadInst>(Val)->setArgOperand(Idx, V->Val);
+}
+
+CatchSwitchInst *CatchPadInst::getCatchSwitch() const {
+  return cast<CatchSwitchInst>(
+      Ctx.getValue(cast<llvm::CatchPadInst>(Val)->getCatchSwitch()));
+}
+
+CatchPadInst *CatchPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
+                                   BBIterator WhereIt, BasicBlock *WhereBB,
+                                   Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  SmallVector<llvm::Value *> LLVMArgs;
+  LLVMArgs.reserve(Args.size());
+  for (auto *Arg : Args)
+    LLVMArgs.push_back(Arg->Val);
+  llvm::CatchPadInst *LLVMI =
+      Builder.CreateCatchPad(ParentPad->Val, LLVMArgs, Name);
+  return Ctx.createCatchPadInst(LLVMI);
+}
+
+CleanupPadInst *CleanupPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
+                                       BBIterator WhereIt, BasicBlock *WhereBB,
+                                       Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  SmallVector<llvm::Value *> LLVMArgs;
+  LLVMArgs.reserve(Args.size());
+  for (auto *Arg : Args)
+    LLVMArgs.push_back(Arg->Val);
+  llvm::CleanupPadInst *LLVMI =
+      Builder.CreateCleanupPad(ParentPad->Val, LLVMArgs, Name);
+  return Ctx.createCleanupPadInst(LLVMI);
+}
+
+CatchReturnInst *CatchReturnInst::create(CatchPadInst *CatchPad, BasicBlock *BB,
+                                         BBIterator WhereIt,
+                                         BasicBlock *WhereBB, Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::CatchReturnInst *LLVMI = Builder.CreateCatchRet(
+      cast<llvm::CatchPadInst>(CatchPad->Val), cast<llvm::BasicBlock>(BB->Val));
+  return Ctx.createCatchReturnInst(LLVMI);
+}
+
+CatchPadInst *CatchReturnInst::getCatchPad() const {
+  return cast<CatchPadInst>(
+      Ctx.getValue(cast<llvm::CatchReturnInst>(Val)->getCatchPad()));
+}
+
+void CatchReturnInst::setCatchPad(CatchPadInst *CatchPad) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CatchReturnInst::getCatchPad,
+                                       &CatchReturnInst::setCatchPad>>(this);
+  cast<llvm::CatchReturnInst>(Val)->setCatchPad(
+      cast<llvm::CatchPadInst>(CatchPad->Val));
+}
+
+BasicBlock *CatchReturnInst::getSuccessor() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::CatchReturnInst>(Val)->getSuccessor()));
+}
+
+void CatchReturnInst::setSuccessor(BasicBlock *NewSucc) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CatchReturnInst::getSuccessor,
+                                       &CatchReturnInst::setSuccessor>>(this);
+  cast<llvm::CatchReturnInst>(Val)->setSuccessor(
+      cast<llvm::BasicBlock>(NewSucc->Val));
+}
+
+Value *CatchReturnInst::getCatchSwitchParentPad() const {
+  return Ctx.getValue(
+      cast<llvm::CatchReturnInst>(Val)->getCatchSwitchParentPad());
+}
+
+CleanupReturnInst *CleanupReturnInst::create(CleanupPadInst *CleanupPad,
+                                             BasicBlock *UnwindBB,
+                                             BBIterator WhereIt,
+                                             BasicBlock *WhereBB,
+                                             Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  auto *LLVMUnwindBB =
+      UnwindBB != nullptr ? cast<llvm::BasicBlock>(UnwindBB->Val) : nullptr;
+  llvm::CleanupReturnInst *LLVMI = Builder.CreateCleanupRet(
+      cast<llvm::CleanupPadInst>(CleanupPad->Val), LLVMUnwindBB);
+  return Ctx.createCleanupReturnInst(LLVMI);
+}
+
+CleanupPadInst *CleanupReturnInst::getCleanupPad() const {
+  return cast<CleanupPadInst>(
+      Ctx.getValue(cast<llvm::CleanupReturnInst>(Val)->getCleanupPad()));
+}
+
+void CleanupReturnInst::setCleanupPad(CleanupPadInst *CleanupPad) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CleanupReturnInst::getCleanupPad,
+                                       &CleanupReturnInst::setCleanupPad>>(
+          this);
+  cast<llvm::CleanupReturnInst>(Val)->setCleanupPad(
+      cast<llvm::CleanupPadInst>(CleanupPad->Val));
+}
+
+BasicBlock *CleanupReturnInst::getUnwindDest() const {
+  return cast_or_null<BasicBlock>(
+      Ctx.getValue(cast<llvm::CleanupReturnInst>(Val)->getUnwindDest()));
+}
+
+void CleanupReturnInst::setUnwindDest(BasicBlock *NewDest) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CleanupReturnInst::getUnwindDest,
+                                       &CleanupReturnInst::setUnwindDest>>(
+          this);
+  cast<llvm::CleanupReturnInst>(Val)->setUnwindDest(
+      cast<llvm::BasicBlock>(NewDest->Val));
+}
+
+Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
+                                 ArrayRef<Value *> IdxList,
+                                 BasicBlock::iterator WhereIt,
+                                 BasicBlock *WhereBB, Context &Ctx,
+                                 const Twine &NameStr) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  SmallVector<llvm::Value *> LLVMIdxList;
+  LLVMIdxList.reserve(IdxList.size());
+  for (Value *Idx : IdxList)
+    LLVMIdxList.push_back(Idx->Val);
+  llvm::Value *NewV =
+      Builder.CreateGEP(Ty->LLVMTy, Ptr->Val, LLVMIdxList, NameStr);
+  if (auto *NewGEP = dyn_cast<llvm::GetElementPtrInst>(NewV))
+    return Ctx.createGetElementPtrInst(NewGEP);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
+                                 ArrayRef<Value *> IdxList,
+                                 Instruction *InsertBefore, Context &Ctx,
+                                 const Twine &NameStr) {
+  return GetElementPtrInst::create(Ty, Ptr, IdxList,
+                                   InsertBefore->getIterator(),
+                                   InsertBefore->getParent(), Ctx, NameStr);
+}
+
+Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
+                                 ArrayRef<Value *> IdxList,
+                                 BasicBlock *InsertAtEnd, Context &Ctx,
+                                 const Twine &NameStr) {
+  return GetElementPtrInst::create(Ty, Ptr, IdxList, InsertAtEnd->end(),
+                                   InsertAtEnd, Ctx, NameStr);
+}
+
+Type *GetElementPtrInst::getSourceElementType() const {
+  return Ctx.getType(
+      cast<llvm::GetElementPtrInst>(Val)->getSourceElementType());
+}
+
+Type *GetElementPtrInst::getResultElementType() const {
+  return Ctx.getType(
+      cast<llvm::GetElementPtrInst>(Val)->getResultElementType());
+}
+
+Value *GetElementPtrInst::getPointerOperand() const {
+  return Ctx.getValue(cast<llvm::GetElementPtrInst>(Val)->getPointerOperand());
+}
+
+Type *GetElementPtrInst::getPointerOperandType() const {
+  return Ctx.getType(
+      cast<llvm::GetElementPtrInst>(Val)->getPointerOperandType());
+}
+
+BasicBlock *PHINode::LLVMBBToBB::operator()(llvm::BasicBlock *LLVMBB) const {
+  return cast<BasicBlock>(Ctx.getValue(LLVMBB));
+}
+
+PHINode *PHINode::create(Type *Ty, unsigned NumReservedValues,
+                         Instruction *InsertBefore, Context &Ctx,
+                         const Twine &Name) {
+  llvm::PHINode *NewPHI = llvm::PHINode::Create(
+      Ty->LLVMTy, NumReservedValues, Name,
+      InsertBefore->getTopmostLLVMInstruction()->getIterator());
+  return Ctx.createPHINode(NewPHI);
+}
+
+bool PHINode::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::PHI;
+}
+
+Value *PHINode::getIncomingValue(unsigned Idx) const {
+  return Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingValue(Idx));
+}
+void PHINode::setIncomingValue(unsigned Idx, Value *V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&PHINode::getIncomingValue,
+                                              &PHINode::setIncomingValue>>(this,
+                                                                           Idx);
+  cast<llvm::PHINode>(Val)->setIncomingValue(Idx, V->Val);
+}
+BasicBlock *PHINode::getIncomingBlock(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingBlock(Idx)));
+}
+BasicBlock *PHINode::getIncomingBlock(const Use &U) const {
+  llvm::Use *LLVMUse = U.LLVMUse;
+  llvm::BasicBlock *BB = cast<llvm::PHINode>(Val)->getIncomingBlock(*LLVMUse);
+  return cast<BasicBlock>(Ctx.getValue(BB));
+}
+void PHINode::setIncomingBlock(unsigned Idx, BasicBlock *BB) {
+  // Helper to disambiguate PHINode::getIncomingBlock(unsigned).
+  constexpr BasicBlock *(PHINode::*GetIncomingBlockFn)(unsigned) const =
+      &PHINode::getIncomingBlock;
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetterWithIdx<GetIncomingBlockFn, &PHINode::setIncomingBlock>>(
+          this, Idx);
+  cast<llvm::PHINode>(Val)->setIncomingBlock(Idx,
+                                             cast<llvm::BasicBlock>(BB->Val));
+}
+void PHINode::addIncoming(Value *V, BasicBlock *BB) {
+  auto &Tracker = Ctx.getTracker();
+  Tracker.emplaceIfTracking<PHIAddIncoming>(this);
+
+  cast<llvm::PHINode>(Val)->addIncoming(V->Val,
+                                        cast<llvm::BasicBlock>(BB->Val));
+}
+Value *PHINode::removeIncomingValue(unsigned Idx) {
+  auto &Tracker = Ctx.getTracker();
+  Tracker.emplaceIfTracking<PHIRemoveIncoming>(this, Idx);
+  llvm::Value *LLVMV =
+      cast<llvm::PHINode>(Val)->removeIncomingValue(Idx,
+                                                    /*DeletePHIIfEmpty=*/false);
+  return Ctx.getValue(LLVMV);
+}
+Value *PHINode::removeIncomingValue(BasicBlock *BB) {
+  auto &Tracker = Ctx.getTracker();
+  Tracker.emplaceIfTracking<PHIRemoveIncoming>(this, getBasicBlockIndex(BB));
+
+  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
+  llvm::Value *LLVMV =
+      cast<llvm::PHINode>(Val)->removeIncomingValue(LLVMBB,
+                                                    /*DeletePHIIfEmpty=*/false);
+  return Ctx.getValue(LLVMV);
+}
+int PHINode::getBasicBlockIndex(const BasicBlock *BB) const {
+  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
+  return cast<llvm::PHINode>(Val)->getBasicBlockIndex(LLVMBB);
+}
+Value *PHINode::getIncomingValueForBlock(const BasicBlock *BB) const {
+  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
+  llvm::Value *LLVMV =
+      cast<llvm::PHINode>(Val)->getIncomingValueForBlock(LLVMBB);
+  return Ctx.getValue(LLVMV);
+}
+Value *PHINode::hasConstantValue() const {
+  llvm::Value *LLVMV = cast<llvm::PHINode>(Val)->hasConstantValue();
+  return LLVMV != nullptr ? Ctx.getValue(LLVMV) : nullptr;
+}
+void PHINode::replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
+  assert(New && Old && "Sandbox IR PHI node got a null basic block!");
+  for (unsigned Idx = 0, NumOps = cast<llvm::PHINode>(Val)->getNumOperands();
+       Idx != NumOps; ++Idx)
+    if (getIncomingBlock(Idx) == Old)
+      setIncomingBlock(Idx, New);
+}
+void PHINode::removeIncomingValueIf(function_ref<bool(unsigned)> Predicate) {
+  // Avoid duplicate tracking by going through this->removeIncomingValue here at
+  // the expense of some performance. Copy PHI::removeIncomingValueIf more
+  // directly if performance becomes an issue.
+
+  // Removing the element at index X, moves the element previously at X + 1
+  // to X. Working from the end avoids complications from that.
+  unsigned Idx = getNumIncomingValues();
+  while (Idx > 0) {
+    if (Predicate(Idx - 1))
+      removeIncomingValue(Idx - 1);
+    --Idx;
+  }
+}
+
+static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::ZExt:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::ZExt);
+  case Instruction::Opcode::SExt:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::SExt);
+  case Instruction::Opcode::FPToUI:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPToUI);
+  case Instruction::Opcode::FPToSI:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPToSI);
+  case Instruction::Opcode::FPExt:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPExt);
+  case Instruction::Opcode::PtrToInt:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::PtrToInt);
+  case Instruction::Opcode::IntToPtr:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::IntToPtr);
+  case Instruction::Opcode::SIToFP:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::SIToFP);
+  case Instruction::Opcode::UIToFP:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::UIToFP);
+  case Instruction::Opcode::Trunc:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::Trunc);
+  case Instruction::Opcode::FPTrunc:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPTrunc);
+  case Instruction::Opcode::BitCast:
+    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::BitCast);
+  case Instruction::Opcode::AddrSpaceCast:
+    return static_cast<llvm::Instruction::CastOps>(
+        llvm::Instruction::AddrSpaceCast);
+  default:
+    llvm_unreachable("Opcode not suitable for CastInst!");
+  }
+}
+
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::UnaryOps getLLVMUnaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::FNeg:
+    return static_cast<llvm::Instruction::UnaryOps>(llvm::Instruction::FNeg);
+  default:
+    llvm_unreachable("Not a unary op!");
+  }
+}
+
+CatchSwitchInst *CatchSwitchInst::create(Value *ParentPad, BasicBlock *UnwindBB,
+                                         unsigned NumHandlers,
+                                         BBIterator WhereIt,
+                                         BasicBlock *WhereBB, Context &Ctx,
+                                         const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::CatchSwitchInst *LLVMCSI = Builder.CreateCatchSwitch(
+      ParentPad->Val, cast<llvm::BasicBlock>(UnwindBB->Val), NumHandlers, Name);
+  return Ctx.createCatchSwitchInst(LLVMCSI);
+}
+
+Value *CatchSwitchInst::getParentPad() const {
+  return Ctx.getValue(cast<llvm::CatchSwitchInst>(Val)->getParentPad());
+}
+
+void CatchSwitchInst::setParentPad(Value *ParentPad) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CatchSwitchInst::getParentPad,
+                                       &CatchSwitchInst::setParentPad>>(this);
+  cast<llvm::CatchSwitchInst>(Val)->setParentPad(ParentPad->Val);
+}
+
+BasicBlock *CatchSwitchInst::getUnwindDest() const {
+  return cast_or_null<BasicBlock>(
+      Ctx.getValue(cast<llvm::CatchSwitchInst>(Val)->getUnwindDest()));
+}
+
+void CatchSwitchInst::setUnwindDest(BasicBlock *UnwindDest) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&CatchSwitchInst::getUnwindDest,
+                                       &CatchSwitchInst::setUnwindDest>>(this);
+  cast<llvm::CatchSwitchInst>(Val)->setUnwindDest(
+      cast<llvm::BasicBlock>(UnwindDest->Val));
+}
+
+void CatchSwitchInst::addHandler(BasicBlock *Dest) {
+  Ctx.getTracker().emplaceIfTracking<CatchSwitchAddHandler>(this);
+  cast<llvm::CatchSwitchInst>(Val)->addHandler(
+      cast<llvm::BasicBlock>(Dest->Val));
+}
+
+ResumeInst *ResumeInst::create(Value *Exn, BBIterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  auto *LLVMI = cast<llvm::ResumeInst>(Builder.CreateResume(Exn->Val));
+  return Ctx.createResumeInst(LLVMI);
+}
+
+Value *ResumeInst::getValue() const {
+  return Ctx.getValue(cast<llvm::ResumeInst>(Val)->getValue());
+}
+
+SwitchInst *SwitchInst::create(Value *V, BasicBlock *Dest, unsigned NumCases,
+                               BasicBlock::iterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::SwitchInst *LLVMSwitch =
+      Builder.CreateSwitch(V->Val, cast<llvm::BasicBlock>(Dest->Val), NumCases);
+  return Ctx.createSwitchInst(LLVMSwitch);
+}
+
+Value *SwitchInst::getCondition() const {
+  return Ctx.getValue(cast<llvm::SwitchInst>(Val)->getCondition());
+}
+
+void SwitchInst::setCondition(Value *V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&SwitchInst::getCondition, &SwitchInst::setCondition>>(
+          this);
+  cast<llvm::SwitchInst>(Val)->setCondition(V->Val);
+}
+
+BasicBlock *SwitchInst::getDefaultDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getDefaultDest()));
+}
+
+void SwitchInst::setDefaultDest(BasicBlock *DefaultCase) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&SwitchInst::getDefaultDest,
+                                       &SwitchInst::setDefaultDest>>(this);
+  cast<llvm::SwitchInst>(Val)->setDefaultDest(
+      cast<llvm::BasicBlock>(DefaultCase->Val));
+}
+ConstantInt *SwitchInst::findCaseDest(BasicBlock *BB) {
+  auto *LLVMC = cast<llvm::SwitchInst>(Val)->findCaseDest(
+      cast<llvm::BasicBlock>(BB->Val));
+  return LLVMC != nullptr ? cast<ConstantInt>(Ctx.getValue(LLVMC)) : nullptr;
+}
+
+void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
+  Ctx.getTracker().emplaceIfTracking<SwitchAddCase>(this, OnVal);
+  // TODO: Track this!
+  cast<llvm::SwitchInst>(Val)->addCase(cast<llvm::ConstantInt>(OnVal->Val),
+                                       cast<llvm::BasicBlock>(Dest->Val));
+}
+
+SwitchInst::CaseIt SwitchInst::removeCase(CaseIt It) {
+  auto &Case = *It;
+  Ctx.getTracker().emplaceIfTracking<SwitchRemoveCase>(
+      this, Case.getCaseValue(), Case.getCaseSuccessor());
+
+  auto *LLVMSwitch = cast<llvm::SwitchInst>(Val);
+  unsigned CaseNum = It - case_begin();
+  llvm::SwitchInst::CaseIt LLVMIt(LLVMSwitch, CaseNum);
+  auto LLVMCaseIt = LLVMSwitch->removeCase(LLVMIt);
+  unsigned Num = LLVMCaseIt - LLVMSwitch->case_begin();
+  return CaseIt(this, Num);
+}
+
+BasicBlock *SwitchInst::getSuccessor(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getSuccessor(Idx)));
+}
+
+void SwitchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&SwitchInst::getSuccessor,
+                                              &SwitchInst::setSuccessor>>(this,
+                                                                          Idx);
+  cast<llvm::SwitchInst>(Val)->setSuccessor(
+      Idx, cast<llvm::BasicBlock>(NewSucc->Val));
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BBIterator WhereIt, BasicBlock *WhereBB,
+                             Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *NewLLVMV = Builder.CreateUnOp(getLLVMUnaryOp(Op), OpV->Val, Name);
+  if (auto *NewUnOpV = dyn_cast<llvm::UnaryOperator>(NewLLVMV)) {
+    return Ctx.createUnaryOperator(NewUnOpV);
+  }
+  assert(isa<llvm::Constant>(NewLLVMV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewLLVMV));
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             Instruction *InsertBefore, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertBefore->getIterator(), InsertBefore->getParent(),
+                Ctx, Name);
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BasicBlock *InsertAfter, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertAfter->end(), InsertAfter, Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom, BBIterator WhereIt,
+                                            BasicBlock *WhereBB, Context &Ctx,
+                                            const Twine &Name) {
+  auto *NewV = create(Op, OpV, WhereIt, WhereBB, Ctx, Name);
+  if (auto *UnI = dyn_cast<llvm::UnaryOperator>(NewV->Val))
+    UnI->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            Instruction *InsertBefore,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            BasicBlock *InsertAtEnd,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::BinaryOps getLLVMBinaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::Add:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Add);
+  case Instruction::Opcode::FAdd:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FAdd);
+  case Instruction::Opcode::Sub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Sub);
+  case Instruction::Opcode::FSub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FSub);
+  case Instruction::Opcode::Mul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Mul);
+  case Instruction::Opcode::FMul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FMul);
+  case Instruction::Opcode::UDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::UDiv);
+  case Instruction::Opcode::SDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SDiv);
+  case Instruction::Opcode::FDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FDiv);
+  case Instruction::Opcode::URem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::URem);
+  case Instruction::Opcode::SRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SRem);
+  case Instruction::Opcode::FRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FRem);
+  case Instruction::Opcode::Shl:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Shl);
+  case Instruction::Opcode::LShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::LShr);
+  case Instruction::Opcode::AShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::AShr);
+  case Instruction::Opcode::And:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::And);
+  case Instruction::Opcode::Or:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Or);
+  case Instruction::Opcode::Xor:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Xor);
+  default:
+    llvm_unreachable("Not a binary op!");
+  }
+}
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BBIterator WhereIt, BasicBlock *WhereBB,
+                              Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  llvm::Value *NewV =
+      Builder.CreateBinOp(getLLVMBinaryOp(Op), LHS->Val, RHS->Val, Name);
+  if (auto *NewBinOp = dyn_cast<llvm::BinaryOperator>(NewV))
+    return Ctx.createBinaryOperator(NewBinOp);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              Instruction *InsertBefore, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BasicBlock *InsertAtEnd, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertAtEnd->end(), InsertAtEnd, Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BBIterator WhereIt,
+                                             BasicBlock *WhereBB, Context &Ctx,
+                                             const Twine &Name) {
+
+  Value *NewV = create(Op, LHS, RHS, WhereIt, WhereBB, Ctx, Name);
+  if (auto *NewBO = dyn_cast<BinaryOperator>(NewV))
+    cast<llvm::BinaryOperator>(NewBO->Val)->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             Instruction *InsertBefore,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom,
+                               InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BasicBlock *InsertAtEnd,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
+void PossiblyDisjointInst::setIsDisjoint(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&PossiblyDisjointInst::isDisjoint,
+                                       &PossiblyDisjointInst::setIsDisjoint>>(
+          this);
+  cast<llvm::PossiblyDisjointInst>(Val)->setIsDisjoint(B);
+}
+
+void AtomicRMWInst::setAlignment(Align Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getAlign,
+                                       &AtomicRMWInst::setAlignment>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setAlignment(Align);
+}
+
+void AtomicRMWInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::isVolatile,
+                                       &AtomicRMWInst::setVolatile>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setVolatile(V);
+}
+
+void AtomicRMWInst::setOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getOrdering,
+                                       &AtomicRMWInst::setOrdering>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setOrdering(Ordering);
+}
+
+void AtomicRMWInst::setSyncScopeID(SyncScope::ID SSID) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getSyncScopeID,
+                                       &AtomicRMWInst::setSyncScopeID>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setSyncScopeID(SSID);
+}
+
+Value *AtomicRMWInst::getPointerOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getPointerOperand());
+}
+
+Value *AtomicRMWInst::getValOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getValOperand());
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BBIterator WhereIt, BasicBlock *WhereBB,
+                                     Context &Ctx, SyncScope::ID SSID,
+                                     const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *LLVMAtomicRMW =
+      Builder.CreateAtomicRMW(Op, Ptr->Val, Val->Val, Align, Ordering, SSID);
+  LLVMAtomicRMW->setName(Name);
+  return Ctx.createAtomicRMWInst(LLVMAtomicRMW);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     Instruction *InsertBefore, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, SSID, Name);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BasicBlock *InsertAtEnd, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertAtEnd->end(), InsertAtEnd,
+                Ctx, SSID, Name);
+}
+
+void AtomicCmpXchgInst::setSyncScopeID(SyncScope::ID SSID) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSyncScopeID,
+                                       &AtomicCmpXchgInst::setSyncScopeID>>(
+          this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setSyncScopeID(SSID);
+}
+
+Value *AtomicCmpXchgInst::getPointerOperand() {
+  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getPointerOperand());
+}
+
+Value *AtomicCmpXchgInst::getCompareOperand() {
+  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getCompareOperand());
+}
+
+Value *AtomicCmpXchgInst::getNewValOperand() {
+  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getNewValOperand());
+}
+
+AtomicCmpXchgInst *
+AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
+                          AtomicOrdering SuccessOrdering,
+                          AtomicOrdering FailureOrdering, BBIterator WhereIt,
+                          BasicBlock *WhereBB, Context &Ctx, SyncScope::ID SSID,
+                          const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *LLVMAtomicCmpXchg =
+      Builder.CreateAtomicCmpXchg(Ptr->Val, Cmp->Val, New->Val, Align,
+                                  SuccessOrdering, FailureOrdering, SSID);
+  LLVMAtomicCmpXchg->setName(Name);
+  return Ctx.createAtomicCmpXchgInst(LLVMAtomicCmpXchg);
+}
+
+AtomicCmpXchgInst *AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New,
+                                             MaybeAlign Align,
+                                             AtomicOrdering SuccessOrdering,
+                                             AtomicOrdering FailureOrdering,
+                                             Instruction *InsertBefore,
+                                             Context &Ctx, SyncScope::ID SSID,
+                                             const Twine &Name) {
+  return create(Ptr, Cmp, New, Align, SuccessOrdering, FailureOrdering,
+                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
+                SSID, Name);
+}
+
+AtomicCmpXchgInst *AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New,
+                                             MaybeAlign Align,
+                                             AtomicOrdering SuccessOrdering,
+                                             AtomicOrdering FailureOrdering,
+                                             BasicBlock *InsertAtEnd,
+                                             Context &Ctx, SyncScope::ID SSID,
+                                             const Twine &Name) {
+  return create(Ptr, Cmp, New, Align, SuccessOrdering, FailureOrdering,
+                InsertAtEnd->end(), InsertAtEnd, Ctx, SSID, Name);
+}
+
+void AtomicCmpXchgInst::setAlignment(Align Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getAlign,
+                                       &AtomicCmpXchgInst::setAlignment>>(this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setAlignment(Align);
+}
+
+void AtomicCmpXchgInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::isVolatile,
+                                       &AtomicCmpXchgInst::setVolatile>>(this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setVolatile(V);
+}
+
+void AtomicCmpXchgInst::setWeak(bool IsWeak) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::isWeak,
+                                       &AtomicCmpXchgInst::setWeak>>(this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setWeak(IsWeak);
+}
+
+void AtomicCmpXchgInst::setSuccessOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSuccessOrdering,
+                                       &AtomicCmpXchgInst::setSuccessOrdering>>(
+          this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setSuccessOrdering(Ordering);
+}
+
+void AtomicCmpXchgInst::setFailureOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getFailureOrdering,
+                                       &AtomicCmpXchgInst::setFailureOrdering>>(
+          this);
+  cast<llvm::AtomicCmpXchgInst>(Val)->setFailureOrdering(Ordering);
+}
+
+AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace, BBIterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               Value *ArraySize, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *NewAlloca =
+      Builder.CreateAlloca(Ty->LLVMTy, AddrSpace, ArraySize->Val, Name);
+  return Ctx.createAllocaInst(NewAlloca);
+}
+
+AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace,
+                               Instruction *InsertBefore, Context &Ctx,
+                               Value *ArraySize, const Twine &Name) {
+  return create(Ty, AddrSpace, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, ArraySize, Name);
+}
+
+AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace,
+                               BasicBlock *InsertAtEnd, Context &Ctx,
+                               Value *ArraySize, const Twine &Name) {
+  return create(Ty, AddrSpace, InsertAtEnd->end(), InsertAtEnd, Ctx, ArraySize,
+                Name);
+}
+
+Type *AllocaInst::getAllocatedType() const {
+  return Ctx.getType(cast<llvm::AllocaInst>(Val)->getAllocatedType());
+}
+
+void AllocaInst::setAllocatedType(Type *Ty) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AllocaInst::getAllocatedType,
+                                       &AllocaInst::setAllocatedType>>(this);
+  cast<llvm::AllocaInst>(Val)->setAllocatedType(Ty->LLVMTy);
+}
+
+void AllocaInst::setAlignment(Align Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&AllocaInst::getAlign, &AllocaInst::setAlignment>>(
+          this);
+  cast<llvm::AllocaInst>(Val)->setAlignment(Align);
+}
+
+void AllocaInst::setUsedWithInAlloca(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AllocaInst::isUsedWithInAlloca,
+                                       &AllocaInst::setUsedWithInAlloca>>(this);
+  cast<llvm::AllocaInst>(Val)->setUsedWithInAlloca(V);
+}
+
+Value *AllocaInst::getArraySize() {
+  return Ctx.getValue(cast<llvm::AllocaInst>(Val)->getArraySize());
+}
+
+PointerType *AllocaInst::getType() const {
+  return cast<PointerType>(Ctx.getType(cast<llvm::AllocaInst>(Val)->getType()));
+}
+
+Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
+                        BBIterator WhereIt, BasicBlock *WhereBB, Context &Ctx,
+                        const Twine &Name) {
+  assert(getLLVMCastOp(Op) && "Opcode not suitable for CastInst!");
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *NewV =
+      Builder.CreateCast(getLLVMCastOp(Op), Operand->Val, DestTy->LLVMTy, Name);
+  if (auto *NewCI = dyn_cast<llvm::CastInst>(NewV))
+    return Ctx.createCastInst(NewCI);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
+                        Instruction *InsertBefore, Context &Ctx,
+                        const Twine &Name) {
+  return create(DestTy, Op, Operand, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
+                        BasicBlock *InsertAtEnd, Context &Ctx,
+                        const Twine &Name) {
+  return create(DestTy, Op, Operand, InsertAtEnd->end(), InsertAtEnd, Ctx,
+                Name);
+}
+
+bool CastInst::classof(const Value *From) {
+  return From->getSubclassID() == ClassID::Cast;
+}
+
+Type *CastInst::getSrcTy() const {
+  return Ctx.getType(cast<llvm::CastInst>(Val)->getSrcTy());
+}
+
+Type *CastInst::getDestTy() const {
+  return Ctx.getType(cast<llvm::CastInst>(Val)->getDestTy());
+}
+
+void PossiblyNonNegInst::setNonNeg(bool B) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&PossiblyNonNegInst::hasNonNeg,
+                                       &PossiblyNonNegInst::setNonNeg>>(this);
+  cast<llvm::PossiblyNonNegInst>(Val)->setNonNeg(B);
+}
+
+Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
+                                 Instruction *InsertBefore, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  llvm::Value *NewV =
+      Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
+  if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
+    return Ctx.createInsertElementInst(NewInsert);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
+                                 BasicBlock *InsertAtEnd, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::Value *NewV =
+      Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
+  if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
+    return Ctx.createInsertElementInst(NewInsert);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ExtractElementInst::create(Value *Vec, Value *Idx,
+                                  Instruction *InsertBefore, Context &Ctx,
+                                  const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
+  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
+    return Ctx.createExtractElementInst(NewExtract);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ExtractElementInst::create(Value *Vec, Value *Idx,
+                                  BasicBlock *InsertAtEnd, Context &Ctx,
+                                  const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
+  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
+    return Ctx.createExtractElementInst(NewExtract);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
+                                 Instruction *InsertBefore, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  llvm::Value *NewV =
+      Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
+  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
+    return Ctx.createShuffleVectorInst(NewShuffle);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
+                                 BasicBlock *InsertAtEnd, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::Value *NewV =
+      Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
+  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
+    return Ctx.createShuffleVectorInst(NewShuffle);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
+                                 Instruction *InsertBefore, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
+  llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
+  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
+    return Ctx.createShuffleVectorInst(NewShuffle);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
+                                 BasicBlock *InsertAtEnd, Context &Ctx,
+                                 const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
+  llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
+  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
+    return Ctx.createShuffleVectorInst(NewShuffle);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
+  Ctx.getTracker().emplaceIfTracking<ShuffleVectorSetMask>(this);
+  cast<llvm::ShuffleVectorInst>(Val)->setShuffleMask(Mask);
+}
+
+VectorType *ShuffleVectorInst::getType() const {
+  return cast<VectorType>(
+      Ctx.getType(cast<llvm::ShuffleVectorInst>(Val)->getType()));
+}
+
+void ShuffleVectorInst::commute() {
+  Ctx.getTracker().emplaceIfTracking<ShuffleVectorSetMask>(this);
+  Ctx.getTracker().emplaceIfTracking<UseSwap>(getOperandUse(0),
+                                              getOperandUse(1));
+  cast<llvm::ShuffleVectorInst>(Val)->commute();
+}
+
+Constant *ShuffleVectorInst::getShuffleMaskForBitcode() const {
+  return Ctx.getOrCreateConstant(
+      cast<llvm::ShuffleVectorInst>(Val)->getShuffleMaskForBitcode());
+}
+
+Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
+                                                          Type *ResultTy) {
+  return ResultTy->getContext().getOrCreateConstant(
+      llvm::ShuffleVectorInst::convertShuffleMaskForBitcode(Mask,
+                                                            ResultTy->LLVMTy));
+}
+
+VectorType *ExtractElementInst::getVectorOperandType() const {
+  return cast<VectorType>(Ctx.getType(getVectorOperand()->getType()->LLVMTy));
+}
+
+Value *ExtractValueInst::create(Value *Agg, ArrayRef<unsigned> Idxs,
+                                BBIterator WhereIt, BasicBlock *WhereBB,
+                                Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::Value *NewV = Builder.CreateExtractValue(Agg->Val, Idxs, Name);
+  if (auto *NewExtractValueInst = dyn_cast<llvm::ExtractValueInst>(NewV))
+    return Ctx.createExtractValueInst(NewExtractValueInst);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Type *ExtractValueInst::getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs) {
+  auto *LLVMTy = llvm::ExtractValueInst::getIndexedType(Agg->LLVMTy, Idxs);
+  return Agg->getContext().getType(LLVMTy);
+}
+
+Value *InsertValueInst::create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
+                               BBIterator WhereIt, BasicBlock *WhereBB,
+                               Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::Value *NewV = Builder.CreateInsertValue(Agg->Val, Val->Val, Idxs, Name);
+  if (auto *NewInsertValueInst = dyn_cast<llvm::InsertValueInst>(NewV))
+    return Ctx.createInsertValueInst(NewInsertValueInst);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+ConstantTokenNone *ConstantTokenNone::get(Context &Ctx) {
+  auto *LLVMC = llvm::ConstantTokenNone::get(Ctx.LLVMCtx);
+  return cast<ConstantTokenNone>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Module.cpp b/llvm/lib/SandboxIR/Module.cpp
new file mode 100644
index 0000000000000..a6a5fb2aae8a1
--- /dev/null
+++ b/llvm/lib/SandboxIR/Module.cpp
@@ -0,0 +1,42 @@
+//===- Module.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Module.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Value.h"
+
+using namespace llvm::sandboxir;
+
+Function *Module::getFunction(StringRef Name) const {
+  llvm::Function *LLVMF = LLVMM.getFunction(Name);
+  return cast_or_null<Function>(Ctx.getValue(LLVMF));
+}
+
+GlobalVariable *Module::getGlobalVariable(StringRef Name,
+                                          bool AllowInternal) const {
+  return cast_or_null<GlobalVariable>(
+      Ctx.getValue(LLVMM.getGlobalVariable(Name, AllowInternal)));
+}
+
+GlobalAlias *Module::getNamedAlias(StringRef Name) const {
+  return cast_or_null<GlobalAlias>(Ctx.getValue(LLVMM.getNamedAlias(Name)));
+}
+
+GlobalIFunc *Module::getNamedIFunc(StringRef Name) const {
+  return cast_or_null<GlobalIFunc>(Ctx.getValue(LLVMM.getNamedIFunc(Name)));
+}
+
+#ifndef NDEBUG
+void Module::dumpOS(raw_ostream &OS) const { OS << LLVMM; }
+
+void Module::dump() const {
+  dumpOS(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
diff --git a/llvm/lib/SandboxIR/PassManager.cpp b/llvm/lib/SandboxIR/PassManager.cpp
index 4abd39b28e87a..4168420a01ce2 100644
--- a/llvm/lib/SandboxIR/PassManager.cpp
+++ b/llvm/lib/SandboxIR/PassManager.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/PassManager.h"
-#include "llvm/SandboxIR/SandboxIR.h"
 
 using namespace llvm::sandboxir;
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Region.cpp b/llvm/lib/SandboxIR/Region.cpp
similarity index 96%
rename from llvm/lib/Transforms/Vectorize/SandboxVectorizer/Region.cpp
rename to llvm/lib/SandboxIR/Region.cpp
index 5f2c28484f62b..b14c87f44260f 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Region.cpp
+++ b/llvm/lib/SandboxIR/Region.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Region.h"
+#include "llvm/SandboxIR/Region.h"
 
 namespace llvm::sandboxir {
 
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 124c1bf92ca7f..89575da2578db 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -6,10 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/SandboxIR/SandboxIR.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/SandboxIR/Argument.h"
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/User.h"
 #include "llvm/Support/Debug.h"
 #include <sstream>
 
@@ -104,3427 +107,3 @@ int OperandUseIterator::operator-(const OperandUseIterator &Other) const {
   int OtherOpNo = Other.Use.getOperandNo();
   return ThisOpNo - OtherOpNo;
 }
-
-Value::Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx)
-    : SubclassID(SubclassID), Val(Val), Ctx(Ctx) {
-#ifndef NDEBUG
-  UID = Ctx.getNumValues();
-#endif
-}
-
-Value::use_iterator Value::use_begin() {
-  llvm::Use *LLVMUse = nullptr;
-  if (Val->use_begin() != Val->use_end())
-    LLVMUse = &*Val->use_begin();
-  User *User = LLVMUse != nullptr ? cast_or_null<sandboxir::User>(Ctx.getValue(
-                                        Val->use_begin()->getUser()))
-                                  : nullptr;
-  return use_iterator(Use(LLVMUse, User, Ctx));
-}
-
-Value::user_iterator Value::user_begin() {
-  auto UseBegin = Val->use_begin();
-  auto UseEnd = Val->use_end();
-  bool AtEnd = UseBegin == UseEnd;
-  llvm::Use *LLVMUse = AtEnd ? nullptr : &*UseBegin;
-  User *User =
-      AtEnd ? nullptr
-            : cast_or_null<sandboxir::User>(Ctx.getValue(&*LLVMUse->getUser()));
-  return user_iterator(Use(LLVMUse, User, Ctx), UseToUser());
-}
-
-unsigned Value::getNumUses() const { return range_size(Val->users()); }
-
-Type *Value::getType() const { return Ctx.getType(Val->getType()); }
-
-void Value::replaceUsesWithIf(
-    Value *OtherV, llvm::function_ref<bool(const Use &)> ShouldReplace) {
-  assert(getType() == OtherV->getType() && "Can't replace with different type");
-  llvm::Value *OtherVal = OtherV->Val;
-  // We are delegating RUWIf to LLVM IR's RUWIf.
-  Val->replaceUsesWithIf(
-      OtherVal, [&ShouldReplace, this](llvm::Use &LLVMUse) -> bool {
-        User *DstU = cast_or_null<User>(Ctx.getValue(LLVMUse.getUser()));
-        if (DstU == nullptr)
-          return false;
-        Use UseToReplace(&LLVMUse, DstU, Ctx);
-        if (!ShouldReplace(UseToReplace))
-          return false;
-        Ctx.getTracker().emplaceIfTracking<UseSet>(UseToReplace);
-        return true;
-      });
-}
-
-void Value::replaceAllUsesWith(Value *Other) {
-  assert(getType() == Other->getType() &&
-         "Replacing with Value of different type!");
-  auto &Tracker = Ctx.getTracker();
-  if (Tracker.isTracking()) {
-    for (auto Use : uses())
-      Tracker.track(std::make_unique<UseSet>(Use));
-  }
-  // We are delegating RAUW to LLVM IR's RAUW.
-  Val->replaceAllUsesWith(Other->Val);
-}
-
-#ifndef NDEBUG
-std::string Value::getUid() const {
-  std::stringstream SS;
-  SS << "SB" << UID << ".";
-  return SS.str();
-}
-
-void Value::dumpCommonHeader(raw_ostream &OS) const {
-  OS << getUid() << " " << getSubclassIDStr(SubclassID) << " ";
-}
-
-void Value::dumpCommonFooter(raw_ostream &OS) const {
-  OS.indent(2) << "Val: ";
-  if (Val)
-    OS << *Val;
-  else
-    OS << "NULL";
-  OS << "\n";
-}
-
-void Value::dumpCommonPrefix(raw_ostream &OS) const {
-  if (Val)
-    OS << *Val;
-  else
-    OS << "NULL ";
-}
-
-void Value::dumpCommonSuffix(raw_ostream &OS) const {
-  OS << " ; " << getUid() << " (" << getSubclassIDStr(SubclassID) << ")";
-}
-
-void Value::printAsOperandCommon(raw_ostream &OS) const {
-  if (Val)
-    Val->printAsOperand(OS);
-  else
-    OS << "NULL ";
-}
-
-void Value::dump() const {
-  dumpOS(dbgs());
-  dbgs() << "\n";
-}
-
-void Argument::printAsOperand(raw_ostream &OS) const {
-  printAsOperandCommon(OS);
-}
-void Argument::dumpOS(raw_ostream &OS) const {
-  dumpCommonPrefix(OS);
-  dumpCommonSuffix(OS);
-}
-#endif // NDEBUG
-
-Use User::getOperandUseDefault(unsigned OpIdx, bool Verify) const {
-  assert((!Verify || OpIdx < getNumOperands()) && "Out of bounds!");
-  assert(isa<llvm::User>(Val) && "Non-users have no operands!");
-  llvm::Use *LLVMUse;
-  if (OpIdx != getNumOperands())
-    LLVMUse = &cast<llvm::User>(Val)->getOperandUse(OpIdx);
-  else
-    LLVMUse = cast<llvm::User>(Val)->op_end();
-  return Use(LLVMUse, const_cast<User *>(this), Ctx);
-}
-
-#ifndef NDEBUG
-void User::verifyUserOfLLVMUse(const llvm::Use &Use) const {
-  assert(Ctx.getValue(Use.getUser()) == this &&
-         "Use not found in this SBUser's operands!");
-}
-#endif
-
-bool User::classof(const Value *From) {
-  switch (From->getSubclassID()) {
-#define DEF_VALUE(ID, CLASS)
-#define DEF_USER(ID, CLASS)                                                    \
-  case ClassID::ID:                                                            \
-    return true;
-#define DEF_INSTR(ID, OPC, CLASS)                                              \
-  case ClassID::ID:                                                            \
-    return true;
-#include "llvm/SandboxIR/SandboxIRValues.def"
-  default:
-    return false;
-  }
-}
-
-void User::setOperand(unsigned OperandIdx, Value *Operand) {
-  assert(isa<llvm::User>(Val) && "No operands!");
-  Ctx.getTracker().emplaceIfTracking<UseSet>(getOperandUse(OperandIdx));
-  // We are delegating to llvm::User::setOperand().
-  cast<llvm::User>(Val)->setOperand(OperandIdx, Operand->Val);
-}
-
-bool User::replaceUsesOfWith(Value *FromV, Value *ToV) {
-  auto &Tracker = Ctx.getTracker();
-  if (Tracker.isTracking()) {
-    for (auto OpIdx : seq<unsigned>(0, getNumOperands())) {
-      auto Use = getOperandUse(OpIdx);
-      if (Use.get() == FromV)
-        Tracker.emplaceIfTracking<UseSet>(Use);
-    }
-  }
-  // We are delegating RUOW to LLVM IR's RUOW.
-  return cast<llvm::User>(Val)->replaceUsesOfWith(FromV->Val, ToV->Val);
-}
-
-#ifndef NDEBUG
-void User::dumpCommonHeader(raw_ostream &OS) const {
-  Value::dumpCommonHeader(OS);
-  // TODO: This is incomplete
-}
-#endif // NDEBUG
-
-BBIterator &BBIterator::operator++() {
-  auto ItE = BB->end();
-  assert(It != ItE && "Already at end!");
-  ++It;
-  if (It == ItE)
-    return *this;
-  Instruction &NextI = *cast<sandboxir::Instruction>(Ctx->getValue(&*It));
-  unsigned Num = NextI.getNumOfIRInstrs();
-  assert(Num > 0 && "Bad getNumOfIRInstrs()");
-  It = std::next(It, Num - 1);
-  return *this;
-}
-
-BBIterator &BBIterator::operator--() {
-  assert(It != BB->begin() && "Already at begin!");
-  if (It == BB->end()) {
-    --It;
-    return *this;
-  }
-  Instruction &CurrI = **this;
-  unsigned Num = CurrI.getNumOfIRInstrs();
-  assert(Num > 0 && "Bad getNumOfIRInstrs()");
-  assert(std::prev(It, Num - 1) != BB->begin() && "Already at begin!");
-  It = std::prev(It, Num);
-  return *this;
-}
-
-BasicBlock *BBIterator::getNodeParent() const {
-  llvm::BasicBlock *Parent = const_cast<BBIterator *>(this)->It.getNodeParent();
-  return cast<BasicBlock>(Ctx->getValue(Parent));
-}
-
-const char *Instruction::getOpcodeName(Opcode Opc) {
-  switch (Opc) {
-#define OP(OPC)                                                                \
-  case Opcode::OPC:                                                            \
-    return #OPC;
-#define OPCODES(...) __VA_ARGS__
-#define DEF_INSTR(ID, OPC, CLASS) OPC
-#include "llvm/SandboxIR/SandboxIRValues.def"
-  }
-  llvm_unreachable("Unknown Opcode");
-}
-
-llvm::Instruction *Instruction::getTopmostLLVMInstruction() const {
-  Instruction *Prev = getPrevNode();
-  if (Prev == nullptr) {
-    // If at top of the BB, return the first BB instruction.
-    return &*cast<llvm::BasicBlock>(getParent()->Val)->begin();
-  }
-  // Else get the Previous sandbox IR instruction's bottom IR instruction and
-  // return its successor.
-  llvm::Instruction *PrevBotI = cast<llvm::Instruction>(Prev->Val);
-  return PrevBotI->getNextNode();
-}
-
-BBIterator Instruction::getIterator() const {
-  auto *I = cast<llvm::Instruction>(Val);
-  return BasicBlock::iterator(I->getParent(), I->getIterator(), &Ctx);
-}
-
-Instruction *Instruction::getNextNode() const {
-  assert(getParent() != nullptr && "Detached!");
-  assert(getIterator() != getParent()->end() && "Already at end!");
-  // `Val` is the bottom-most LLVM IR instruction. Get the next in the chain,
-  // and get the corresponding sandboxir Instruction that maps to it. This works
-  // even for SandboxIR Instructions that map to more than one LLVM Instruction.
-  auto *LLVMI = cast<llvm::Instruction>(Val);
-  assert(LLVMI->getParent() != nullptr && "LLVM IR instr is detached!");
-  auto *NextLLVMI = LLVMI->getNextNode();
-  auto *NextI = cast_or_null<Instruction>(Ctx.getValue(NextLLVMI));
-  if (NextI == nullptr)
-    return nullptr;
-  return NextI;
-}
-
-Instruction *Instruction::getPrevNode() const {
-  assert(getParent() != nullptr && "Detached!");
-  auto It = getIterator();
-  if (It != getParent()->begin())
-    return std::prev(getIterator()).get();
-  return nullptr;
-}
-
-void Instruction::removeFromParent() {
-  Ctx.getTracker().emplaceIfTracking<RemoveFromParent>(this);
-
-  // Detach all the LLVM IR instructions from their parent BB.
-  for (llvm::Instruction *I : getLLVMInstrs())
-    I->removeFromParent();
-}
-
-void Instruction::eraseFromParent() {
-  assert(users().empty() && "Still connected to users, can't erase!");
-  std::unique_ptr<Value> Detached = Ctx.detach(this);
-  auto LLVMInstrs = getLLVMInstrs();
-
-  auto &Tracker = Ctx.getTracker();
-  if (Tracker.isTracking()) {
-    Tracker.track(std::make_unique<EraseFromParent>(std::move(Detached)));
-    // We don't actually delete the IR instruction, because then it would be
-    // impossible to bring it back from the dead at the same memory location.
-    // Instead we remove it from its BB and track its current location.
-    for (llvm::Instruction *I : LLVMInstrs)
-      I->removeFromParent();
-    // TODO: Multi-instructions need special treatment because some of the
-    // references are internal to the instruction.
-    for (llvm::Instruction *I : LLVMInstrs)
-      I->dropAllReferences();
-  } else {
-    // Erase in reverse to avoid erasing nstructions with attached uses.
-    for (llvm::Instruction *I : reverse(LLVMInstrs))
-      I->eraseFromParent();
-  }
-}
-
-void Instruction::moveBefore(BasicBlock &BB, const BBIterator &WhereIt) {
-  if (std::next(getIterator()) == WhereIt)
-    // Destination is same as origin, nothing to do.
-    return;
-
-  Ctx.getTracker().emplaceIfTracking<MoveInstr>(this);
-
-  auto *LLVMBB = cast<llvm::BasicBlock>(BB.Val);
-  llvm::BasicBlock::iterator It;
-  if (WhereIt == BB.end()) {
-    It = LLVMBB->end();
-  } else {
-    Instruction *WhereI = &*WhereIt;
-    It = WhereI->getTopmostLLVMInstruction()->getIterator();
-  }
-  // TODO: Move this to the verifier of sandboxir::Instruction.
-  assert(is_sorted(getLLVMInstrs(),
-                   [](auto *I1, auto *I2) { return I1->comesBefore(I2); }) &&
-         "Expected program order!");
-  // Do the actual move in LLVM IR.
-  for (auto *I : getLLVMInstrs())
-    I->moveBefore(*LLVMBB, It);
-}
-
-void Instruction::insertBefore(Instruction *BeforeI) {
-  llvm::Instruction *BeforeTopI = BeforeI->getTopmostLLVMInstruction();
-  // TODO: Move this to the verifier of sandboxir::Instruction.
-  assert(is_sorted(getLLVMInstrs(),
-                   [](auto *I1, auto *I2) { return I1->comesBefore(I2); }) &&
-         "Expected program order!");
-
-  Ctx.getTracker().emplaceIfTracking<InsertIntoBB>(this);
-
-  // Insert the LLVM IR Instructions in program order.
-  for (llvm::Instruction *I : getLLVMInstrs())
-    I->insertBefore(BeforeTopI);
-}
-
-void Instruction::insertAfter(Instruction *AfterI) {
-  insertInto(AfterI->getParent(), std::next(AfterI->getIterator()));
-}
-
-void Instruction::insertInto(BasicBlock *BB, const BBIterator &WhereIt) {
-  llvm::BasicBlock *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
-  llvm::Instruction *LLVMBeforeI;
-  llvm::BasicBlock::iterator LLVMBeforeIt;
-  Instruction *BeforeI;
-  if (WhereIt != BB->end()) {
-    BeforeI = &*WhereIt;
-    LLVMBeforeI = BeforeI->getTopmostLLVMInstruction();
-    LLVMBeforeIt = LLVMBeforeI->getIterator();
-  } else {
-    BeforeI = nullptr;
-    LLVMBeforeI = nullptr;
-    LLVMBeforeIt = LLVMBB->end();
-  }
-
-  Ctx.getTracker().emplaceIfTracking<InsertIntoBB>(this);
-
-  // Insert the LLVM IR Instructions in program order.
-  for (llvm::Instruction *I : getLLVMInstrs())
-    I->insertInto(LLVMBB, LLVMBeforeIt);
-}
-
-BasicBlock *Instruction::getParent() const {
-  // Get the LLVM IR Instruction that this maps to, get its parent, and get the
-  // corresponding sandboxir::BasicBlock by looking it up in sandboxir::Context.
-  auto *BB = cast<llvm::Instruction>(Val)->getParent();
-  if (BB == nullptr)
-    return nullptr;
-  return cast<BasicBlock>(Ctx.getValue(BB));
-}
-
-bool Instruction::classof(const sandboxir::Value *From) {
-  switch (From->getSubclassID()) {
-#define DEF_INSTR(ID, OPC, CLASS)                                              \
-  case ClassID::ID:                                                            \
-    return true;
-#include "llvm/SandboxIR/SandboxIRValues.def"
-  default:
-    return false;
-  }
-}
-
-void Instruction::setHasNoUnsignedWrap(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasNoUnsignedWrap,
-                                       &Instruction::setHasNoUnsignedWrap>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasNoUnsignedWrap(B);
-}
-
-void Instruction::setHasNoSignedWrap(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasNoSignedWrap,
-                                       &Instruction::setHasNoSignedWrap>>(this);
-  cast<llvm::Instruction>(Val)->setHasNoSignedWrap(B);
-}
-
-void Instruction::setFast(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&Instruction::isFast, &Instruction::setFast>>(this);
-  cast<llvm::Instruction>(Val)->setFast(B);
-}
-
-void Instruction::setIsExact(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&Instruction::isExact, &Instruction::setIsExact>>(this);
-  cast<llvm::Instruction>(Val)->setIsExact(B);
-}
-
-void Instruction::setHasAllowReassoc(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowReassoc,
-                                       &Instruction::setHasAllowReassoc>>(this);
-  cast<llvm::Instruction>(Val)->setHasAllowReassoc(B);
-}
-
-void Instruction::setHasNoNaNs(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&Instruction::hasNoNaNs, &Instruction::setHasNoNaNs>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasNoNaNs(B);
-}
-
-void Instruction::setHasNoInfs(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&Instruction::hasNoInfs, &Instruction::setHasNoInfs>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasNoInfs(B);
-}
-
-void Instruction::setHasNoSignedZeros(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasNoSignedZeros,
-                                       &Instruction::setHasNoSignedZeros>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasNoSignedZeros(B);
-}
-
-void Instruction::setHasAllowReciprocal(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowReciprocal,
-                                       &Instruction::setHasAllowReciprocal>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasAllowReciprocal(B);
-}
-
-void Instruction::setHasAllowContract(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasAllowContract,
-                                       &Instruction::setHasAllowContract>>(
-          this);
-  cast<llvm::Instruction>(Val)->setHasAllowContract(B);
-}
-
-void Instruction::setFastMathFlags(FastMathFlags FMF) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::getFastMathFlags,
-                                       &Instruction::copyFastMathFlags>>(this);
-  cast<llvm::Instruction>(Val)->setFastMathFlags(FMF);
-}
-
-void Instruction::copyFastMathFlags(FastMathFlags FMF) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::getFastMathFlags,
-                                       &Instruction::copyFastMathFlags>>(this);
-  cast<llvm::Instruction>(Val)->copyFastMathFlags(FMF);
-}
-
-Type *Instruction::getAccessType() const {
-  return Ctx.getType(cast<llvm::Instruction>(Val)->getAccessType());
-}
-
-void Instruction::setHasApproxFunc(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&Instruction::hasApproxFunc,
-                                       &Instruction::setHasApproxFunc>>(this);
-  cast<llvm::Instruction>(Val)->setHasApproxFunc(B);
-}
-
-#ifndef NDEBUG
-void Instruction::dumpOS(raw_ostream &OS) const {
-  OS << "Unimplemented! Please override dump().";
-}
-#endif // NDEBUG
-
-VAArgInst *VAArgInst::create(Value *List, Type *Ty, BBIterator WhereIt,
-                             BasicBlock *WhereBB, Context &Ctx,
-                             const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  auto *LLVMI =
-      cast<llvm::VAArgInst>(Builder.CreateVAArg(List->Val, Ty->LLVMTy, Name));
-  return Ctx.createVAArgInst(LLVMI);
-}
-
-Value *VAArgInst::getPointerOperand() {
-  return Ctx.getValue(cast<llvm::VAArgInst>(Val)->getPointerOperand());
-}
-
-FreezeInst *FreezeInst::create(Value *V, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  auto *LLVMI = cast<llvm::FreezeInst>(Builder.CreateFreeze(V->Val, Name));
-  return Ctx.createFreezeInst(LLVMI);
-}
-
-FenceInst *FenceInst::create(AtomicOrdering Ordering, BBIterator WhereIt,
-                             BasicBlock *WhereBB, Context &Ctx,
-                             SyncScope::ID SSID) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::FenceInst *LLVMI = Builder.CreateFence(Ordering, SSID);
-  return Ctx.createFenceInst(LLVMI);
-}
-
-void FenceInst::setOrdering(AtomicOrdering Ordering) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&FenceInst::getOrdering, &FenceInst::setOrdering>>(
-          this);
-  cast<llvm::FenceInst>(Val)->setOrdering(Ordering);
-}
-
-void FenceInst::setSyncScopeID(SyncScope::ID SSID) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&FenceInst::getSyncScopeID,
-                                       &FenceInst::setSyncScopeID>>(this);
-  cast<llvm::FenceInst>(Val)->setSyncScopeID(SSID);
-}
-
-Value *SelectInst::createCommon(Value *Cond, Value *True, Value *False,
-                                const Twine &Name, IRBuilder<> &Builder,
-                                Context &Ctx) {
-  llvm::Value *NewV =
-      Builder.CreateSelect(Cond->Val, True->Val, False->Val, Name);
-  if (auto *NewSI = dyn_cast<llvm::SelectInst>(NewV))
-    return Ctx.createSelectInst(NewSI);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *SelectInst::create(Value *Cond, Value *True, Value *False,
-                          Instruction *InsertBefore, Context &Ctx,
-                          const Twine &Name) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  return createCommon(Cond, True, False, Name, Builder, Ctx);
-}
-
-Value *SelectInst::create(Value *Cond, Value *True, Value *False,
-                          BasicBlock *InsertAtEnd, Context &Ctx,
-                          const Twine &Name) {
-  auto *IRInsertAtEnd = cast<llvm::BasicBlock>(InsertAtEnd->Val);
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(IRInsertAtEnd);
-  return createCommon(Cond, True, False, Name, Builder, Ctx);
-}
-
-void SelectInst::swapValues() {
-  Ctx.getTracker().emplaceIfTracking<UseSwap>(getOperandUse(1),
-                                              getOperandUse(2));
-  cast<llvm::SelectInst>(Val)->swapValues();
-}
-
-bool SelectInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Select;
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, Instruction *InsertBefore,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(cast<llvm::Instruction>(LLVMBefore));
-  llvm::BranchInst *NewBr =
-      Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *InsertAtEnd,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::BranchInst *NewBr =
-      Builder.CreateBr(cast<llvm::BasicBlock>(IfTrue->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                               Value *Cond, Instruction *InsertBefore,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(LLVMBefore);
-  llvm::BranchInst *NewBr =
-      Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
-                           cast<llvm::BasicBlock>(IfFalse->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-BranchInst *BranchInst::create(BasicBlock *IfTrue, BasicBlock *IfFalse,
-                               Value *Cond, BasicBlock *InsertAtEnd,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::BranchInst *NewBr =
-      Builder.CreateCondBr(Cond->Val, cast<llvm::BasicBlock>(IfTrue->Val),
-                           cast<llvm::BasicBlock>(IfFalse->Val));
-  return Ctx.createBranchInst(NewBr);
-}
-
-bool BranchInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Br;
-}
-
-Value *BranchInst::getCondition() const {
-  assert(isConditional() && "Cannot get condition of an uncond branch!");
-  return Ctx.getValue(cast<llvm::BranchInst>(Val)->getCondition());
-}
-
-BasicBlock *BranchInst::getSuccessor(unsigned SuccIdx) const {
-  assert(SuccIdx < getNumSuccessors() &&
-         "Successor # out of range for Branch!");
-  return cast_or_null<BasicBlock>(
-      Ctx.getValue(cast<llvm::BranchInst>(Val)->getSuccessor(SuccIdx)));
-}
-
-void BranchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
-  assert((Idx == 0 || Idx == 1) && "Out of bounds!");
-  setOperand(2u - Idx, NewSucc);
-}
-
-BasicBlock *BranchInst::LLVMBBToSBBB::operator()(llvm::BasicBlock *BB) const {
-  return cast<BasicBlock>(Ctx.getValue(BB));
-}
-const BasicBlock *
-BranchInst::ConstLLVMBBToSBBB::operator()(const llvm::BasicBlock *BB) const {
-  return cast<BasicBlock>(Ctx.getValue(BB));
-}
-
-void LoadInst::setVolatile(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&LoadInst::isVolatile, &LoadInst::setVolatile>>(this);
-  cast<llvm::LoadInst>(Val)->setVolatile(V);
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, Context &Ctx,
-                           const Twine &Name) {
-  return create(Ty, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx, Name);
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           Instruction *InsertBefore, bool IsVolatile,
-                           Context &Ctx, const Twine &Name) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  auto *NewLI =
-      Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
-  auto *NewSBI = Ctx.createLoadInst(NewLI);
-  return NewSBI;
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, Context &Ctx,
-                           const Twine &Name) {
-  return create(Ty, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx, Name);
-}
-
-LoadInst *LoadInst::create(Type *Ty, Value *Ptr, MaybeAlign Align,
-                           BasicBlock *InsertAtEnd, bool IsVolatile,
-                           Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  auto *NewLI =
-      Builder.CreateAlignedLoad(Ty->LLVMTy, Ptr->Val, Align, IsVolatile, Name);
-  auto *NewSBI = Ctx.createLoadInst(NewLI);
-  return NewSBI;
-}
-
-bool LoadInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Load;
-}
-
-Value *LoadInst::getPointerOperand() const {
-  return Ctx.getValue(cast<llvm::LoadInst>(Val)->getPointerOperand());
-}
-
-void StoreInst::setVolatile(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&StoreInst::isVolatile, &StoreInst::setVolatile>>(this);
-  cast<llvm::StoreInst>(Val)->setVolatile(V);
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             Instruction *InsertBefore, Context &Ctx) {
-  return create(V, Ptr, Align, InsertBefore, /*IsVolatile=*/false, Ctx);
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             Instruction *InsertBefore, bool IsVolatile,
-                             Context &Ctx) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
-  auto *NewSBI = Ctx.createStoreInst(NewSI);
-  return NewSBI;
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             BasicBlock *InsertAtEnd, Context &Ctx) {
-  return create(V, Ptr, Align, InsertAtEnd, /*IsVolatile=*/false, Ctx);
-}
-
-StoreInst *StoreInst::create(Value *V, Value *Ptr, MaybeAlign Align,
-                             BasicBlock *InsertAtEnd, bool IsVolatile,
-                             Context &Ctx) {
-  auto *InsertAtEndIR = cast<llvm::BasicBlock>(InsertAtEnd->Val);
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertAtEndIR);
-  auto *NewSI = Builder.CreateAlignedStore(V->Val, Ptr->Val, Align, IsVolatile);
-  auto *NewSBI = Ctx.createStoreInst(NewSI);
-  return NewSBI;
-}
-
-bool StoreInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Store;
-}
-
-Value *StoreInst::getValueOperand() const {
-  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getValueOperand());
-}
-
-Value *StoreInst::getPointerOperand() const {
-  return Ctx.getValue(cast<llvm::StoreInst>(Val)->getPointerOperand());
-}
-
-UnreachableInst *UnreachableInst::create(Instruction *InsertBefore,
-                                         Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  llvm::Instruction *LLVMBefore = InsertBefore->getTopmostLLVMInstruction();
-  Builder.SetInsertPoint(LLVMBefore);
-  llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
-  return Ctx.createUnreachableInst(NewUI);
-}
-
-UnreachableInst *UnreachableInst::create(BasicBlock *InsertAtEnd,
-                                         Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::UnreachableInst *NewUI = Builder.CreateUnreachable();
-  return Ctx.createUnreachableInst(NewUI);
-}
-
-bool UnreachableInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Unreachable;
-}
-
-ReturnInst *ReturnInst::createCommon(Value *RetVal, IRBuilder<> &Builder,
-                                     Context &Ctx) {
-  llvm::ReturnInst *NewRI;
-  if (RetVal != nullptr)
-    NewRI = Builder.CreateRet(RetVal->Val);
-  else
-    NewRI = Builder.CreateRetVoid();
-  return Ctx.createReturnInst(NewRI);
-}
-
-ReturnInst *ReturnInst::create(Value *RetVal, Instruction *InsertBefore,
-                               Context &Ctx) {
-  llvm::Instruction *BeforeIR = InsertBefore->getTopmostLLVMInstruction();
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(BeforeIR);
-  return createCommon(RetVal, Builder, Ctx);
-}
-
-ReturnInst *ReturnInst::create(Value *RetVal, BasicBlock *InsertAtEnd,
-                               Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  return createCommon(RetVal, Builder, Ctx);
-}
-
-Value *ReturnInst::getReturnValue() const {
-  auto *LLVMRetVal = cast<llvm::ReturnInst>(Val)->getReturnValue();
-  return LLVMRetVal != nullptr ? Ctx.getValue(LLVMRetVal) : nullptr;
-}
-
-FunctionType *CallBase::getFunctionType() const {
-  return cast<FunctionType>(
-      Ctx.getType(cast<llvm::CallBase>(Val)->getFunctionType()));
-}
-
-Value *CallBase::getCalledOperand() const {
-  return Ctx.getValue(cast<llvm::CallBase>(Val)->getCalledOperand());
-}
-
-Use CallBase::getCalledOperandUse() const {
-  llvm::Use *LLVMUse = &cast<llvm::CallBase>(Val)->getCalledOperandUse();
-  return Use(LLVMUse, cast<User>(Ctx.getValue(LLVMUse->getUser())), Ctx);
-}
-
-Function *CallBase::getCalledFunction() const {
-  return cast_or_null<Function>(
-      Ctx.getValue(cast<llvm::CallBase>(Val)->getCalledFunction()));
-}
-Function *CallBase::getCaller() {
-  return cast<Function>(Ctx.getValue(cast<llvm::CallBase>(Val)->getCaller()));
-}
-
-void CallBase::setCalledFunction(Function *F) {
-  // F's function type is private, so we rely on `setCalledFunction()` to update
-  // it. But even though we are calling `setCalledFunction()` we also need to
-  // track this change at the SandboxIR level, which is why we call
-  // `setCalledOperand()` here.
-  // Note: This may break if `setCalledFunction()` early returns if `F`
-  // is already set, but we do have a unit test for it.
-  setCalledOperand(F);
-  cast<llvm::CallBase>(Val)->setCalledFunction(
-      cast<llvm::FunctionType>(F->getFunctionType()->LLVMTy),
-      cast<llvm::Function>(F->Val));
-}
-
-CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, BasicBlock::iterator WhereIt,
-                           BasicBlock *WhereBB, Context &Ctx,
-                           const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  SmallVector<llvm::Value *> LLVMArgs;
-  LLVMArgs.reserve(Args.size());
-  for (Value *Arg : Args)
-    LLVMArgs.push_back(Arg->Val);
-  llvm::CallInst *NewCI = Builder.CreateCall(
-      cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val, LLVMArgs, NameStr);
-  return Ctx.createCallInst(NewCI);
-}
-
-CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, Instruction *InsertBefore,
-                           Context &Ctx, const Twine &NameStr) {
-  return CallInst::create(FTy, Func, Args, InsertBefore->getIterator(),
-                          InsertBefore->getParent(), Ctx, NameStr);
-}
-
-CallInst *CallInst::create(FunctionType *FTy, Value *Func,
-                           ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                           Context &Ctx, const Twine &NameStr) {
-  return CallInst::create(FTy, Func, Args, InsertAtEnd->end(), InsertAtEnd, Ctx,
-                          NameStr);
-}
-
-InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  SmallVector<llvm::Value *> LLVMArgs;
-  LLVMArgs.reserve(Args.size());
-  for (Value *Arg : Args)
-    LLVMArgs.push_back(Arg->Val);
-  llvm::InvokeInst *Invoke = Builder.CreateInvoke(
-      cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val,
-      cast<llvm::BasicBlock>(IfNormal->Val),
-      cast<llvm::BasicBlock>(IfException->Val), LLVMArgs, NameStr);
-  return Ctx.createInvokeInst(Invoke);
-}
-
-InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args,
-                               Instruction *InsertBefore, Context &Ctx,
-                               const Twine &NameStr) {
-  return create(FTy, Func, IfNormal, IfException, Args,
-                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
-                NameStr);
-}
-
-InvokeInst *InvokeInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *IfNormal, BasicBlock *IfException,
-                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                               Context &Ctx, const Twine &NameStr) {
-  return create(FTy, Func, IfNormal, IfException, Args, InsertAtEnd->end(),
-                InsertAtEnd, Ctx, NameStr);
-}
-
-BasicBlock *InvokeInst::getNormalDest() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getNormalDest()));
-}
-BasicBlock *InvokeInst::getUnwindDest() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getUnwindDest()));
-}
-void InvokeInst::setNormalDest(BasicBlock *BB) {
-  setOperand(1, BB);
-  assert(getNormalDest() == BB && "LLVM IR uses a different operan index!");
-}
-void InvokeInst::setUnwindDest(BasicBlock *BB) {
-  setOperand(2, BB);
-  assert(getUnwindDest() == BB && "LLVM IR uses a different operan index!");
-}
-LandingPadInst *InvokeInst::getLandingPadInst() const {
-  return cast<LandingPadInst>(
-      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getLandingPadInst()));
-  ;
-}
-BasicBlock *InvokeInst::getSuccessor(unsigned SuccIdx) const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::InvokeInst>(Val)->getSuccessor(SuccIdx)));
-}
-
-CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *DefaultDest,
-                               ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-
-  SmallVector<llvm::BasicBlock *> LLVMIndirectDests;
-  LLVMIndirectDests.reserve(IndirectDests.size());
-  for (BasicBlock *IndDest : IndirectDests)
-    LLVMIndirectDests.push_back(cast<llvm::BasicBlock>(IndDest->Val));
-
-  SmallVector<llvm::Value *> LLVMArgs;
-  LLVMArgs.reserve(Args.size());
-  for (Value *Arg : Args)
-    LLVMArgs.push_back(Arg->Val);
-
-  llvm::CallBrInst *CallBr =
-      Builder.CreateCallBr(cast<llvm::FunctionType>(FTy->LLVMTy), Func->Val,
-                           cast<llvm::BasicBlock>(DefaultDest->Val),
-                           LLVMIndirectDests, LLVMArgs, NameStr);
-  return Ctx.createCallBrInst(CallBr);
-}
-
-CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *DefaultDest,
-                               ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args,
-                               Instruction *InsertBefore, Context &Ctx,
-                               const Twine &NameStr) {
-  return create(FTy, Func, DefaultDest, IndirectDests, Args,
-                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
-                NameStr);
-}
-CallBrInst *CallBrInst::create(FunctionType *FTy, Value *Func,
-                               BasicBlock *DefaultDest,
-                               ArrayRef<BasicBlock *> IndirectDests,
-                               ArrayRef<Value *> Args, BasicBlock *InsertAtEnd,
-                               Context &Ctx, const Twine &NameStr) {
-  return create(FTy, Func, DefaultDest, IndirectDests, Args, InsertAtEnd->end(),
-                InsertAtEnd, Ctx, NameStr);
-}
-
-Value *CallBrInst::getIndirectDestLabel(unsigned Idx) const {
-  return Ctx.getValue(cast<llvm::CallBrInst>(Val)->getIndirectDestLabel(Idx));
-}
-Value *CallBrInst::getIndirectDestLabelUse(unsigned Idx) const {
-  return Ctx.getValue(
-      cast<llvm::CallBrInst>(Val)->getIndirectDestLabelUse(Idx));
-}
-BasicBlock *CallBrInst::getDefaultDest() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getDefaultDest()));
-}
-BasicBlock *CallBrInst::getIndirectDest(unsigned Idx) const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getIndirectDest(Idx)));
-}
-llvm::SmallVector<BasicBlock *, 16> CallBrInst::getIndirectDests() const {
-  SmallVector<BasicBlock *, 16> BBs;
-  for (llvm::BasicBlock *LLVMBB :
-       cast<llvm::CallBrInst>(Val)->getIndirectDests())
-    BBs.push_back(cast<BasicBlock>(Ctx.getValue(LLVMBB)));
-  return BBs;
-}
-void CallBrInst::setDefaultDest(BasicBlock *BB) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CallBrInst::getDefaultDest,
-                                       &CallBrInst::setDefaultDest>>(this);
-  cast<llvm::CallBrInst>(Val)->setDefaultDest(cast<llvm::BasicBlock>(BB->Val));
-}
-void CallBrInst::setIndirectDest(unsigned Idx, BasicBlock *BB) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetterWithIdx<&CallBrInst::getIndirectDest,
-                                              &CallBrInst::setIndirectDest>>(
-          this, Idx);
-  cast<llvm::CallBrInst>(Val)->setIndirectDest(Idx,
-                                               cast<llvm::BasicBlock>(BB->Val));
-}
-BasicBlock *CallBrInst::getSuccessor(unsigned Idx) const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::CallBrInst>(Val)->getSuccessor(Idx)));
-}
-
-LandingPadInst *LandingPadInst::create(Type *RetTy, unsigned NumReservedClauses,
-                                       BBIterator WhereIt, BasicBlock *WhereBB,
-                                       Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::LandingPadInst *LLVMI =
-      Builder.CreateLandingPad(RetTy->LLVMTy, NumReservedClauses, Name);
-  return Ctx.createLandingPadInst(LLVMI);
-}
-
-void LandingPadInst::setCleanup(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&LandingPadInst::isCleanup,
-                                       &LandingPadInst::setCleanup>>(this);
-  cast<llvm::LandingPadInst>(Val)->setCleanup(V);
-}
-
-Constant *LandingPadInst::getClause(unsigned Idx) const {
-  return cast<Constant>(
-      Ctx.getValue(cast<llvm::LandingPadInst>(Val)->getClause(Idx)));
-}
-
-Value *FuncletPadInst::getParentPad() const {
-  return Ctx.getValue(cast<llvm::FuncletPadInst>(Val)->getParentPad());
-}
-
-void FuncletPadInst::setParentPad(Value *ParentPad) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&FuncletPadInst::getParentPad,
-                                       &FuncletPadInst::setParentPad>>(this);
-  cast<llvm::FuncletPadInst>(Val)->setParentPad(ParentPad->Val);
-}
-
-Value *FuncletPadInst::getArgOperand(unsigned Idx) const {
-  return Ctx.getValue(cast<llvm::FuncletPadInst>(Val)->getArgOperand(Idx));
-}
-
-void FuncletPadInst::setArgOperand(unsigned Idx, Value *V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetterWithIdx<&FuncletPadInst::getArgOperand,
-                                              &FuncletPadInst::setArgOperand>>(
-          this, Idx);
-  cast<llvm::FuncletPadInst>(Val)->setArgOperand(Idx, V->Val);
-}
-
-CatchSwitchInst *CatchPadInst::getCatchSwitch() const {
-  return cast<CatchSwitchInst>(
-      Ctx.getValue(cast<llvm::CatchPadInst>(Val)->getCatchSwitch()));
-}
-
-CatchPadInst *CatchPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
-                                   BBIterator WhereIt, BasicBlock *WhereBB,
-                                   Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  SmallVector<llvm::Value *> LLVMArgs;
-  LLVMArgs.reserve(Args.size());
-  for (auto *Arg : Args)
-    LLVMArgs.push_back(Arg->Val);
-  llvm::CatchPadInst *LLVMI =
-      Builder.CreateCatchPad(ParentPad->Val, LLVMArgs, Name);
-  return Ctx.createCatchPadInst(LLVMI);
-}
-
-CleanupPadInst *CleanupPadInst::create(Value *ParentPad, ArrayRef<Value *> Args,
-                                       BBIterator WhereIt, BasicBlock *WhereBB,
-                                       Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  SmallVector<llvm::Value *> LLVMArgs;
-  LLVMArgs.reserve(Args.size());
-  for (auto *Arg : Args)
-    LLVMArgs.push_back(Arg->Val);
-  llvm::CleanupPadInst *LLVMI =
-      Builder.CreateCleanupPad(ParentPad->Val, LLVMArgs, Name);
-  return Ctx.createCleanupPadInst(LLVMI);
-}
-
-CatchReturnInst *CatchReturnInst::create(CatchPadInst *CatchPad, BasicBlock *BB,
-                                         BBIterator WhereIt,
-                                         BasicBlock *WhereBB, Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::CatchReturnInst *LLVMI = Builder.CreateCatchRet(
-      cast<llvm::CatchPadInst>(CatchPad->Val), cast<llvm::BasicBlock>(BB->Val));
-  return Ctx.createCatchReturnInst(LLVMI);
-}
-
-CatchPadInst *CatchReturnInst::getCatchPad() const {
-  return cast<CatchPadInst>(
-      Ctx.getValue(cast<llvm::CatchReturnInst>(Val)->getCatchPad()));
-}
-
-void CatchReturnInst::setCatchPad(CatchPadInst *CatchPad) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CatchReturnInst::getCatchPad,
-                                       &CatchReturnInst::setCatchPad>>(this);
-  cast<llvm::CatchReturnInst>(Val)->setCatchPad(
-      cast<llvm::CatchPadInst>(CatchPad->Val));
-}
-
-BasicBlock *CatchReturnInst::getSuccessor() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::CatchReturnInst>(Val)->getSuccessor()));
-}
-
-void CatchReturnInst::setSuccessor(BasicBlock *NewSucc) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CatchReturnInst::getSuccessor,
-                                       &CatchReturnInst::setSuccessor>>(this);
-  cast<llvm::CatchReturnInst>(Val)->setSuccessor(
-      cast<llvm::BasicBlock>(NewSucc->Val));
-}
-
-Value *CatchReturnInst::getCatchSwitchParentPad() const {
-  return Ctx.getValue(
-      cast<llvm::CatchReturnInst>(Val)->getCatchSwitchParentPad());
-}
-
-CleanupReturnInst *CleanupReturnInst::create(CleanupPadInst *CleanupPad,
-                                             BasicBlock *UnwindBB,
-                                             BBIterator WhereIt,
-                                             BasicBlock *WhereBB,
-                                             Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  auto *LLVMUnwindBB =
-      UnwindBB != nullptr ? cast<llvm::BasicBlock>(UnwindBB->Val) : nullptr;
-  llvm::CleanupReturnInst *LLVMI = Builder.CreateCleanupRet(
-      cast<llvm::CleanupPadInst>(CleanupPad->Val), LLVMUnwindBB);
-  return Ctx.createCleanupReturnInst(LLVMI);
-}
-
-CleanupPadInst *CleanupReturnInst::getCleanupPad() const {
-  return cast<CleanupPadInst>(
-      Ctx.getValue(cast<llvm::CleanupReturnInst>(Val)->getCleanupPad()));
-}
-
-void CleanupReturnInst::setCleanupPad(CleanupPadInst *CleanupPad) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CleanupReturnInst::getCleanupPad,
-                                       &CleanupReturnInst::setCleanupPad>>(
-          this);
-  cast<llvm::CleanupReturnInst>(Val)->setCleanupPad(
-      cast<llvm::CleanupPadInst>(CleanupPad->Val));
-}
-
-BasicBlock *CleanupReturnInst::getUnwindDest() const {
-  return cast_or_null<BasicBlock>(
-      Ctx.getValue(cast<llvm::CleanupReturnInst>(Val)->getUnwindDest()));
-}
-
-void CleanupReturnInst::setUnwindDest(BasicBlock *NewDest) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CleanupReturnInst::getUnwindDest,
-                                       &CleanupReturnInst::setUnwindDest>>(
-          this);
-  cast<llvm::CleanupReturnInst>(Val)->setUnwindDest(
-      cast<llvm::BasicBlock>(NewDest->Val));
-}
-
-Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
-                                 ArrayRef<Value *> IdxList,
-                                 BasicBlock::iterator WhereIt,
-                                 BasicBlock *WhereBB, Context &Ctx,
-                                 const Twine &NameStr) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  SmallVector<llvm::Value *> LLVMIdxList;
-  LLVMIdxList.reserve(IdxList.size());
-  for (Value *Idx : IdxList)
-    LLVMIdxList.push_back(Idx->Val);
-  llvm::Value *NewV =
-      Builder.CreateGEP(Ty->LLVMTy, Ptr->Val, LLVMIdxList, NameStr);
-  if (auto *NewGEP = dyn_cast<llvm::GetElementPtrInst>(NewV))
-    return Ctx.createGetElementPtrInst(NewGEP);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
-                                 ArrayRef<Value *> IdxList,
-                                 Instruction *InsertBefore, Context &Ctx,
-                                 const Twine &NameStr) {
-  return GetElementPtrInst::create(Ty, Ptr, IdxList,
-                                   InsertBefore->getIterator(),
-                                   InsertBefore->getParent(), Ctx, NameStr);
-}
-
-Value *GetElementPtrInst::create(Type *Ty, Value *Ptr,
-                                 ArrayRef<Value *> IdxList,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &NameStr) {
-  return GetElementPtrInst::create(Ty, Ptr, IdxList, InsertAtEnd->end(),
-                                   InsertAtEnd, Ctx, NameStr);
-}
-
-Type *GetElementPtrInst::getSourceElementType() const {
-  return Ctx.getType(
-      cast<llvm::GetElementPtrInst>(Val)->getSourceElementType());
-}
-
-Type *GetElementPtrInst::getResultElementType() const {
-  return Ctx.getType(
-      cast<llvm::GetElementPtrInst>(Val)->getResultElementType());
-}
-
-Value *GetElementPtrInst::getPointerOperand() const {
-  return Ctx.getValue(cast<llvm::GetElementPtrInst>(Val)->getPointerOperand());
-}
-
-Type *GetElementPtrInst::getPointerOperandType() const {
-  return Ctx.getType(
-      cast<llvm::GetElementPtrInst>(Val)->getPointerOperandType());
-}
-
-BasicBlock *PHINode::LLVMBBToBB::operator()(llvm::BasicBlock *LLVMBB) const {
-  return cast<BasicBlock>(Ctx.getValue(LLVMBB));
-}
-
-PHINode *PHINode::create(Type *Ty, unsigned NumReservedValues,
-                         Instruction *InsertBefore, Context &Ctx,
-                         const Twine &Name) {
-  llvm::PHINode *NewPHI = llvm::PHINode::Create(
-      Ty->LLVMTy, NumReservedValues, Name,
-      InsertBefore->getTopmostLLVMInstruction()->getIterator());
-  return Ctx.createPHINode(NewPHI);
-}
-
-bool PHINode::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::PHI;
-}
-
-Value *PHINode::getIncomingValue(unsigned Idx) const {
-  return Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingValue(Idx));
-}
-void PHINode::setIncomingValue(unsigned Idx, Value *V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetterWithIdx<&PHINode::getIncomingValue,
-                                              &PHINode::setIncomingValue>>(this,
-                                                                           Idx);
-  cast<llvm::PHINode>(Val)->setIncomingValue(Idx, V->Val);
-}
-BasicBlock *PHINode::getIncomingBlock(unsigned Idx) const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingBlock(Idx)));
-}
-BasicBlock *PHINode::getIncomingBlock(const Use &U) const {
-  llvm::Use *LLVMUse = U.LLVMUse;
-  llvm::BasicBlock *BB = cast<llvm::PHINode>(Val)->getIncomingBlock(*LLVMUse);
-  return cast<BasicBlock>(Ctx.getValue(BB));
-}
-void PHINode::setIncomingBlock(unsigned Idx, BasicBlock *BB) {
-  // Helper to disambiguate PHINode::getIncomingBlock(unsigned).
-  constexpr BasicBlock *(PHINode::*GetIncomingBlockFn)(unsigned) const =
-      &PHINode::getIncomingBlock;
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetterWithIdx<GetIncomingBlockFn, &PHINode::setIncomingBlock>>(
-          this, Idx);
-  cast<llvm::PHINode>(Val)->setIncomingBlock(Idx,
-                                             cast<llvm::BasicBlock>(BB->Val));
-}
-void PHINode::addIncoming(Value *V, BasicBlock *BB) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHIAddIncoming>(this);
-
-  cast<llvm::PHINode>(Val)->addIncoming(V->Val,
-                                        cast<llvm::BasicBlock>(BB->Val));
-}
-Value *PHINode::removeIncomingValue(unsigned Idx) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHIRemoveIncoming>(this, Idx);
-  llvm::Value *LLVMV =
-      cast<llvm::PHINode>(Val)->removeIncomingValue(Idx,
-                                                    /*DeletePHIIfEmpty=*/false);
-  return Ctx.getValue(LLVMV);
-}
-Value *PHINode::removeIncomingValue(BasicBlock *BB) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHIRemoveIncoming>(this, getBasicBlockIndex(BB));
-
-  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
-  llvm::Value *LLVMV =
-      cast<llvm::PHINode>(Val)->removeIncomingValue(LLVMBB,
-                                                    /*DeletePHIIfEmpty=*/false);
-  return Ctx.getValue(LLVMV);
-}
-int PHINode::getBasicBlockIndex(const BasicBlock *BB) const {
-  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
-  return cast<llvm::PHINode>(Val)->getBasicBlockIndex(LLVMBB);
-}
-Value *PHINode::getIncomingValueForBlock(const BasicBlock *BB) const {
-  auto *LLVMBB = cast<llvm::BasicBlock>(BB->Val);
-  llvm::Value *LLVMV =
-      cast<llvm::PHINode>(Val)->getIncomingValueForBlock(LLVMBB);
-  return Ctx.getValue(LLVMV);
-}
-Value *PHINode::hasConstantValue() const {
-  llvm::Value *LLVMV = cast<llvm::PHINode>(Val)->hasConstantValue();
-  return LLVMV != nullptr ? Ctx.getValue(LLVMV) : nullptr;
-}
-void PHINode::replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) {
-  assert(New && Old && "Sandbox IR PHI node got a null basic block!");
-  for (unsigned Idx = 0, NumOps = cast<llvm::PHINode>(Val)->getNumOperands();
-       Idx != NumOps; ++Idx)
-    if (getIncomingBlock(Idx) == Old)
-      setIncomingBlock(Idx, New);
-}
-void PHINode::removeIncomingValueIf(function_ref<bool(unsigned)> Predicate) {
-  // Avoid duplicate tracking by going through this->removeIncomingValue here at
-  // the expense of some performance. Copy PHI::removeIncomingValueIf more
-  // directly if performance becomes an issue.
-
-  // Removing the element at index X, moves the element previously at X + 1
-  // to X. Working from the end avoids complications from that.
-  unsigned Idx = getNumIncomingValues();
-  while (Idx > 0) {
-    if (Predicate(Idx - 1))
-      removeIncomingValue(Idx - 1);
-    --Idx;
-  }
-}
-
-static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
-  switch (Opc) {
-  case Instruction::Opcode::ZExt:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::ZExt);
-  case Instruction::Opcode::SExt:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::SExt);
-  case Instruction::Opcode::FPToUI:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPToUI);
-  case Instruction::Opcode::FPToSI:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPToSI);
-  case Instruction::Opcode::FPExt:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPExt);
-  case Instruction::Opcode::PtrToInt:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::PtrToInt);
-  case Instruction::Opcode::IntToPtr:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::IntToPtr);
-  case Instruction::Opcode::SIToFP:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::SIToFP);
-  case Instruction::Opcode::UIToFP:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::UIToFP);
-  case Instruction::Opcode::Trunc:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::Trunc);
-  case Instruction::Opcode::FPTrunc:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPTrunc);
-  case Instruction::Opcode::BitCast:
-    return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::BitCast);
-  case Instruction::Opcode::AddrSpaceCast:
-    return static_cast<llvm::Instruction::CastOps>(
-        llvm::Instruction::AddrSpaceCast);
-  default:
-    llvm_unreachable("Opcode not suitable for CastInst!");
-  }
-}
-
-/// \Returns the LLVM opcode that corresponds to \p Opc.
-static llvm::Instruction::UnaryOps getLLVMUnaryOp(Instruction::Opcode Opc) {
-  switch (Opc) {
-  case Instruction::Opcode::FNeg:
-    return static_cast<llvm::Instruction::UnaryOps>(llvm::Instruction::FNeg);
-  default:
-    llvm_unreachable("Not a unary op!");
-  }
-}
-
-CatchSwitchInst *CatchSwitchInst::create(Value *ParentPad, BasicBlock *UnwindBB,
-                                         unsigned NumHandlers,
-                                         BBIterator WhereIt,
-                                         BasicBlock *WhereBB, Context &Ctx,
-                                         const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::CatchSwitchInst *LLVMCSI = Builder.CreateCatchSwitch(
-      ParentPad->Val, cast<llvm::BasicBlock>(UnwindBB->Val), NumHandlers, Name);
-  return Ctx.createCatchSwitchInst(LLVMCSI);
-}
-
-Value *CatchSwitchInst::getParentPad() const {
-  return Ctx.getValue(cast<llvm::CatchSwitchInst>(Val)->getParentPad());
-}
-
-void CatchSwitchInst::setParentPad(Value *ParentPad) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CatchSwitchInst::getParentPad,
-                                       &CatchSwitchInst::setParentPad>>(this);
-  cast<llvm::CatchSwitchInst>(Val)->setParentPad(ParentPad->Val);
-}
-
-BasicBlock *CatchSwitchInst::getUnwindDest() const {
-  return cast_or_null<BasicBlock>(
-      Ctx.getValue(cast<llvm::CatchSwitchInst>(Val)->getUnwindDest()));
-}
-
-void CatchSwitchInst::setUnwindDest(BasicBlock *UnwindDest) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&CatchSwitchInst::getUnwindDest,
-                                       &CatchSwitchInst::setUnwindDest>>(this);
-  cast<llvm::CatchSwitchInst>(Val)->setUnwindDest(
-      cast<llvm::BasicBlock>(UnwindDest->Val));
-}
-
-void CatchSwitchInst::addHandler(BasicBlock *Dest) {
-  Ctx.getTracker().emplaceIfTracking<CatchSwitchAddHandler>(this);
-  cast<llvm::CatchSwitchInst>(Val)->addHandler(
-      cast<llvm::BasicBlock>(Dest->Val));
-}
-
-ResumeInst *ResumeInst::create(Value *Exn, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  auto *LLVMI = cast<llvm::ResumeInst>(Builder.CreateResume(Exn->Val));
-  return Ctx.createResumeInst(LLVMI);
-}
-
-Value *ResumeInst::getValue() const {
-  return Ctx.getValue(cast<llvm::ResumeInst>(Val)->getValue());
-}
-
-SwitchInst *SwitchInst::create(Value *V, BasicBlock *Dest, unsigned NumCases,
-                               BasicBlock::iterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::SwitchInst *LLVMSwitch =
-      Builder.CreateSwitch(V->Val, cast<llvm::BasicBlock>(Dest->Val), NumCases);
-  return Ctx.createSwitchInst(LLVMSwitch);
-}
-
-Value *SwitchInst::getCondition() const {
-  return Ctx.getValue(cast<llvm::SwitchInst>(Val)->getCondition());
-}
-
-void SwitchInst::setCondition(Value *V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&SwitchInst::getCondition, &SwitchInst::setCondition>>(
-          this);
-  cast<llvm::SwitchInst>(Val)->setCondition(V->Val);
-}
-
-BasicBlock *SwitchInst::getDefaultDest() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getDefaultDest()));
-}
-
-void SwitchInst::setDefaultDest(BasicBlock *DefaultCase) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&SwitchInst::getDefaultDest,
-                                       &SwitchInst::setDefaultDest>>(this);
-  cast<llvm::SwitchInst>(Val)->setDefaultDest(
-      cast<llvm::BasicBlock>(DefaultCase->Val));
-}
-ConstantInt *SwitchInst::findCaseDest(BasicBlock *BB) {
-  auto *LLVMC = cast<llvm::SwitchInst>(Val)->findCaseDest(
-      cast<llvm::BasicBlock>(BB->Val));
-  return LLVMC != nullptr ? cast<ConstantInt>(Ctx.getValue(LLVMC)) : nullptr;
-}
-
-void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
-  Ctx.getTracker().emplaceIfTracking<SwitchAddCase>(this, OnVal);
-  // TODO: Track this!
-  cast<llvm::SwitchInst>(Val)->addCase(cast<llvm::ConstantInt>(OnVal->Val),
-                                       cast<llvm::BasicBlock>(Dest->Val));
-}
-
-SwitchInst::CaseIt SwitchInst::removeCase(CaseIt It) {
-  auto &Case = *It;
-  Ctx.getTracker().emplaceIfTracking<SwitchRemoveCase>(
-      this, Case.getCaseValue(), Case.getCaseSuccessor());
-
-  auto *LLVMSwitch = cast<llvm::SwitchInst>(Val);
-  unsigned CaseNum = It - case_begin();
-  llvm::SwitchInst::CaseIt LLVMIt(LLVMSwitch, CaseNum);
-  auto LLVMCaseIt = LLVMSwitch->removeCase(LLVMIt);
-  unsigned Num = LLVMCaseIt - LLVMSwitch->case_begin();
-  return CaseIt(this, Num);
-}
-
-BasicBlock *SwitchInst::getSuccessor(unsigned Idx) const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getSuccessor(Idx)));
-}
-
-void SwitchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetterWithIdx<&SwitchInst::getSuccessor,
-                                              &SwitchInst::setSuccessor>>(this,
-                                                                          Idx);
-  cast<llvm::SwitchInst>(Val)->setSuccessor(
-      Idx, cast<llvm::BasicBlock>(NewSucc->Val));
-}
-
-Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
-                             BBIterator WhereIt, BasicBlock *WhereBB,
-                             Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  auto *NewLLVMV = Builder.CreateUnOp(getLLVMUnaryOp(Op), OpV->Val, Name);
-  if (auto *NewUnOpV = dyn_cast<llvm::UnaryOperator>(NewLLVMV)) {
-    return Ctx.createUnaryOperator(NewUnOpV);
-  }
-  assert(isa<llvm::Constant>(NewLLVMV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewLLVMV));
-}
-
-Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
-                             Instruction *InsertBefore, Context &Ctx,
-                             const Twine &Name) {
-  return create(Op, OpV, InsertBefore->getIterator(), InsertBefore->getParent(),
-                Ctx, Name);
-}
-
-Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
-                             BasicBlock *InsertAfter, Context &Ctx,
-                             const Twine &Name) {
-  return create(Op, OpV, InsertAfter->end(), InsertAfter, Ctx, Name);
-}
-
-Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
-                                            Value *CopyFrom, BBIterator WhereIt,
-                                            BasicBlock *WhereBB, Context &Ctx,
-                                            const Twine &Name) {
-  auto *NewV = create(Op, OpV, WhereIt, WhereBB, Ctx, Name);
-  if (auto *UnI = dyn_cast<llvm::UnaryOperator>(NewV->Val))
-    UnI->copyIRFlags(CopyFrom->Val);
-  return NewV;
-}
-
-Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
-                                            Value *CopyFrom,
-                                            Instruction *InsertBefore,
-                                            Context &Ctx, const Twine &Name) {
-  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertBefore->getIterator(),
-                               InsertBefore->getParent(), Ctx, Name);
-}
-
-Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
-                                            Value *CopyFrom,
-                                            BasicBlock *InsertAtEnd,
-                                            Context &Ctx, const Twine &Name) {
-  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertAtEnd->end(),
-                               InsertAtEnd, Ctx, Name);
-}
-
-/// \Returns the LLVM opcode that corresponds to \p Opc.
-static llvm::Instruction::BinaryOps getLLVMBinaryOp(Instruction::Opcode Opc) {
-  switch (Opc) {
-  case Instruction::Opcode::Add:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Add);
-  case Instruction::Opcode::FAdd:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FAdd);
-  case Instruction::Opcode::Sub:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Sub);
-  case Instruction::Opcode::FSub:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FSub);
-  case Instruction::Opcode::Mul:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Mul);
-  case Instruction::Opcode::FMul:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FMul);
-  case Instruction::Opcode::UDiv:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::UDiv);
-  case Instruction::Opcode::SDiv:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SDiv);
-  case Instruction::Opcode::FDiv:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FDiv);
-  case Instruction::Opcode::URem:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::URem);
-  case Instruction::Opcode::SRem:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SRem);
-  case Instruction::Opcode::FRem:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FRem);
-  case Instruction::Opcode::Shl:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Shl);
-  case Instruction::Opcode::LShr:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::LShr);
-  case Instruction::Opcode::AShr:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::AShr);
-  case Instruction::Opcode::And:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::And);
-  case Instruction::Opcode::Or:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Or);
-  case Instruction::Opcode::Xor:
-    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Xor);
-  default:
-    llvm_unreachable("Not a binary op!");
-  }
-}
-Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
-                              BBIterator WhereIt, BasicBlock *WhereBB,
-                              Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  llvm::Value *NewV =
-      Builder.CreateBinOp(getLLVMBinaryOp(Op), LHS->Val, RHS->Val, Name);
-  if (auto *NewBinOp = dyn_cast<llvm::BinaryOperator>(NewV))
-    return Ctx.createBinaryOperator(NewBinOp);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
-                              Instruction *InsertBefore, Context &Ctx,
-                              const Twine &Name) {
-  return create(Op, LHS, RHS, InsertBefore->getIterator(),
-                InsertBefore->getParent(), Ctx, Name);
-}
-
-Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
-                              BasicBlock *InsertAtEnd, Context &Ctx,
-                              const Twine &Name) {
-  return create(Op, LHS, RHS, InsertAtEnd->end(), InsertAtEnd, Ctx, Name);
-}
-
-Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
-                                             Value *RHS, Value *CopyFrom,
-                                             BBIterator WhereIt,
-                                             BasicBlock *WhereBB, Context &Ctx,
-                                             const Twine &Name) {
-
-  Value *NewV = create(Op, LHS, RHS, WhereIt, WhereBB, Ctx, Name);
-  if (auto *NewBO = dyn_cast<BinaryOperator>(NewV))
-    cast<llvm::BinaryOperator>(NewBO->Val)->copyIRFlags(CopyFrom->Val);
-  return NewV;
-}
-
-Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
-                                             Value *RHS, Value *CopyFrom,
-                                             Instruction *InsertBefore,
-                                             Context &Ctx, const Twine &Name) {
-  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom,
-                               InsertBefore->getIterator(),
-                               InsertBefore->getParent(), Ctx, Name);
-}
-
-Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
-                                             Value *RHS, Value *CopyFrom,
-                                             BasicBlock *InsertAtEnd,
-                                             Context &Ctx, const Twine &Name) {
-  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom, InsertAtEnd->end(),
-                               InsertAtEnd, Ctx, Name);
-}
-
-void PossiblyDisjointInst::setIsDisjoint(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&PossiblyDisjointInst::isDisjoint,
-                                       &PossiblyDisjointInst::setIsDisjoint>>(
-          this);
-  cast<llvm::PossiblyDisjointInst>(Val)->setIsDisjoint(B);
-}
-
-void AtomicRMWInst::setAlignment(Align Align) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getAlign,
-                                       &AtomicRMWInst::setAlignment>>(this);
-  cast<llvm::AtomicRMWInst>(Val)->setAlignment(Align);
-}
-
-void AtomicRMWInst::setVolatile(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::isVolatile,
-                                       &AtomicRMWInst::setVolatile>>(this);
-  cast<llvm::AtomicRMWInst>(Val)->setVolatile(V);
-}
-
-void AtomicRMWInst::setOrdering(AtomicOrdering Ordering) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getOrdering,
-                                       &AtomicRMWInst::setOrdering>>(this);
-  cast<llvm::AtomicRMWInst>(Val)->setOrdering(Ordering);
-}
-
-void AtomicRMWInst::setSyncScopeID(SyncScope::ID SSID) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getSyncScopeID,
-                                       &AtomicRMWInst::setSyncScopeID>>(this);
-  cast<llvm::AtomicRMWInst>(Val)->setSyncScopeID(SSID);
-}
-
-Value *AtomicRMWInst::getPointerOperand() {
-  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getPointerOperand());
-}
-
-Value *AtomicRMWInst::getValOperand() {
-  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getValOperand());
-}
-
-AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
-                                     MaybeAlign Align, AtomicOrdering Ordering,
-                                     BBIterator WhereIt, BasicBlock *WhereBB,
-                                     Context &Ctx, SyncScope::ID SSID,
-                                     const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  auto *LLVMAtomicRMW =
-      Builder.CreateAtomicRMW(Op, Ptr->Val, Val->Val, Align, Ordering, SSID);
-  LLVMAtomicRMW->setName(Name);
-  return Ctx.createAtomicRMWInst(LLVMAtomicRMW);
-}
-
-AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
-                                     MaybeAlign Align, AtomicOrdering Ordering,
-                                     Instruction *InsertBefore, Context &Ctx,
-                                     SyncScope::ID SSID, const Twine &Name) {
-  return create(Op, Ptr, Val, Align, Ordering, InsertBefore->getIterator(),
-                InsertBefore->getParent(), Ctx, SSID, Name);
-}
-
-AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
-                                     MaybeAlign Align, AtomicOrdering Ordering,
-                                     BasicBlock *InsertAtEnd, Context &Ctx,
-                                     SyncScope::ID SSID, const Twine &Name) {
-  return create(Op, Ptr, Val, Align, Ordering, InsertAtEnd->end(), InsertAtEnd,
-                Ctx, SSID, Name);
-}
-
-void AtomicCmpXchgInst::setSyncScopeID(SyncScope::ID SSID) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSyncScopeID,
-                                       &AtomicCmpXchgInst::setSyncScopeID>>(
-          this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setSyncScopeID(SSID);
-}
-
-Value *AtomicCmpXchgInst::getPointerOperand() {
-  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getPointerOperand());
-}
-
-Value *AtomicCmpXchgInst::getCompareOperand() {
-  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getCompareOperand());
-}
-
-Value *AtomicCmpXchgInst::getNewValOperand() {
-  return Ctx.getValue(cast<llvm::AtomicCmpXchgInst>(Val)->getNewValOperand());
-}
-
-AtomicCmpXchgInst *
-AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align,
-                          AtomicOrdering SuccessOrdering,
-                          AtomicOrdering FailureOrdering, BBIterator WhereIt,
-                          BasicBlock *WhereBB, Context &Ctx, SyncScope::ID SSID,
-                          const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  auto *LLVMAtomicCmpXchg =
-      Builder.CreateAtomicCmpXchg(Ptr->Val, Cmp->Val, New->Val, Align,
-                                  SuccessOrdering, FailureOrdering, SSID);
-  LLVMAtomicCmpXchg->setName(Name);
-  return Ctx.createAtomicCmpXchgInst(LLVMAtomicCmpXchg);
-}
-
-AtomicCmpXchgInst *AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New,
-                                             MaybeAlign Align,
-                                             AtomicOrdering SuccessOrdering,
-                                             AtomicOrdering FailureOrdering,
-                                             Instruction *InsertBefore,
-                                             Context &Ctx, SyncScope::ID SSID,
-                                             const Twine &Name) {
-  return create(Ptr, Cmp, New, Align, SuccessOrdering, FailureOrdering,
-                InsertBefore->getIterator(), InsertBefore->getParent(), Ctx,
-                SSID, Name);
-}
-
-AtomicCmpXchgInst *AtomicCmpXchgInst::create(Value *Ptr, Value *Cmp, Value *New,
-                                             MaybeAlign Align,
-                                             AtomicOrdering SuccessOrdering,
-                                             AtomicOrdering FailureOrdering,
-                                             BasicBlock *InsertAtEnd,
-                                             Context &Ctx, SyncScope::ID SSID,
-                                             const Twine &Name) {
-  return create(Ptr, Cmp, New, Align, SuccessOrdering, FailureOrdering,
-                InsertAtEnd->end(), InsertAtEnd, Ctx, SSID, Name);
-}
-
-void AtomicCmpXchgInst::setAlignment(Align Align) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getAlign,
-                                       &AtomicCmpXchgInst::setAlignment>>(this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setAlignment(Align);
-}
-
-void AtomicCmpXchgInst::setVolatile(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::isVolatile,
-                                       &AtomicCmpXchgInst::setVolatile>>(this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setVolatile(V);
-}
-
-void AtomicCmpXchgInst::setWeak(bool IsWeak) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::isWeak,
-                                       &AtomicCmpXchgInst::setWeak>>(this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setWeak(IsWeak);
-}
-
-void AtomicCmpXchgInst::setSuccessOrdering(AtomicOrdering Ordering) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSuccessOrdering,
-                                       &AtomicCmpXchgInst::setSuccessOrdering>>(
-          this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setSuccessOrdering(Ordering);
-}
-
-void AtomicCmpXchgInst::setFailureOrdering(AtomicOrdering Ordering) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getFailureOrdering,
-                                       &AtomicCmpXchgInst::setFailureOrdering>>(
-          this);
-  cast<llvm::AtomicCmpXchgInst>(Val)->setFailureOrdering(Ordering);
-}
-
-AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace, BBIterator WhereIt,
-                               BasicBlock *WhereBB, Context &Ctx,
-                               Value *ArraySize, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  auto *NewAlloca =
-      Builder.CreateAlloca(Ty->LLVMTy, AddrSpace, ArraySize->Val, Name);
-  return Ctx.createAllocaInst(NewAlloca);
-}
-
-AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace,
-                               Instruction *InsertBefore, Context &Ctx,
-                               Value *ArraySize, const Twine &Name) {
-  return create(Ty, AddrSpace, InsertBefore->getIterator(),
-                InsertBefore->getParent(), Ctx, ArraySize, Name);
-}
-
-AllocaInst *AllocaInst::create(Type *Ty, unsigned AddrSpace,
-                               BasicBlock *InsertAtEnd, Context &Ctx,
-                               Value *ArraySize, const Twine &Name) {
-  return create(Ty, AddrSpace, InsertAtEnd->end(), InsertAtEnd, Ctx, ArraySize,
-                Name);
-}
-
-Type *AllocaInst::getAllocatedType() const {
-  return Ctx.getType(cast<llvm::AllocaInst>(Val)->getAllocatedType());
-}
-
-void AllocaInst::setAllocatedType(Type *Ty) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AllocaInst::getAllocatedType,
-                                       &AllocaInst::setAllocatedType>>(this);
-  cast<llvm::AllocaInst>(Val)->setAllocatedType(Ty->LLVMTy);
-}
-
-void AllocaInst::setAlignment(Align Align) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&AllocaInst::getAlign, &AllocaInst::setAlignment>>(
-          this);
-  cast<llvm::AllocaInst>(Val)->setAlignment(Align);
-}
-
-void AllocaInst::setUsedWithInAlloca(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&AllocaInst::isUsedWithInAlloca,
-                                       &AllocaInst::setUsedWithInAlloca>>(this);
-  cast<llvm::AllocaInst>(Val)->setUsedWithInAlloca(V);
-}
-
-Value *AllocaInst::getArraySize() {
-  return Ctx.getValue(cast<llvm::AllocaInst>(Val)->getArraySize());
-}
-
-PointerType *AllocaInst::getType() const {
-  return cast<PointerType>(Ctx.getType(cast<llvm::AllocaInst>(Val)->getType()));
-}
-
-Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
-                        BBIterator WhereIt, BasicBlock *WhereBB, Context &Ctx,
-                        const Twine &Name) {
-  assert(getLLVMCastOp(Op) && "Opcode not suitable for CastInst!");
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt == WhereBB->end())
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  else
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  auto *NewV =
-      Builder.CreateCast(getLLVMCastOp(Op), Operand->Val, DestTy->LLVMTy, Name);
-  if (auto *NewCI = dyn_cast<llvm::CastInst>(NewV))
-    return Ctx.createCastInst(NewCI);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
-                        Instruction *InsertBefore, Context &Ctx,
-                        const Twine &Name) {
-  return create(DestTy, Op, Operand, InsertBefore->getIterator(),
-                InsertBefore->getParent(), Ctx, Name);
-}
-
-Value *CastInst::create(Type *DestTy, Opcode Op, Value *Operand,
-                        BasicBlock *InsertAtEnd, Context &Ctx,
-                        const Twine &Name) {
-  return create(DestTy, Op, Operand, InsertAtEnd->end(), InsertAtEnd, Ctx,
-                Name);
-}
-
-bool CastInst::classof(const Value *From) {
-  return From->getSubclassID() == ClassID::Cast;
-}
-
-Type *CastInst::getSrcTy() const {
-  return Ctx.getType(cast<llvm::CastInst>(Val)->getSrcTy());
-}
-
-Type *CastInst::getDestTy() const {
-  return Ctx.getType(cast<llvm::CastInst>(Val)->getDestTy());
-}
-
-void PossiblyNonNegInst::setNonNeg(bool B) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&PossiblyNonNegInst::hasNonNeg,
-                                       &PossiblyNonNegInst::setNonNeg>>(this);
-  cast<llvm::PossiblyNonNegInst>(Val)->setNonNeg(B);
-}
-
-Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
-                                 Instruction *InsertBefore, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV =
-      Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
-  if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
-    return Ctx.createInsertElementInst(NewInsert);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *InsertElementInst::create(Value *Vec, Value *NewElt, Value *Idx,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV =
-      Builder.CreateInsertElement(Vec->Val, NewElt->Val, Idx->Val, Name);
-  if (auto *NewInsert = dyn_cast<llvm::InsertElementInst>(NewV))
-    return Ctx.createInsertElementInst(NewInsert);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ExtractElementInst::create(Value *Vec, Value *Idx,
-                                  Instruction *InsertBefore, Context &Ctx,
-                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
-  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
-    return Ctx.createExtractElementInst(NewExtract);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ExtractElementInst::create(Value *Vec, Value *Idx,
-                                  BasicBlock *InsertAtEnd, Context &Ctx,
-                                  const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV = Builder.CreateExtractElement(Vec->Val, Idx->Val, Name);
-  if (auto *NewExtract = dyn_cast<llvm::ExtractElementInst>(NewV))
-    return Ctx.createExtractElementInst(NewExtract);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
-                                 Instruction *InsertBefore, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV =
-      Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, Value *Mask,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV =
-      Builder.CreateShuffleVector(V1->Val, V2->Val, Mask->Val, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                                 Instruction *InsertBefore, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Value *ShuffleVectorInst::create(Value *V1, Value *V2, ArrayRef<int> Mask,
-                                 BasicBlock *InsertAtEnd, Context &Ctx,
-                                 const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(cast<llvm::BasicBlock>(InsertAtEnd->Val));
-  llvm::Value *NewV = Builder.CreateShuffleVector(V1->Val, V2->Val, Mask, Name);
-  if (auto *NewShuffle = dyn_cast<llvm::ShuffleVectorInst>(NewV))
-    return Ctx.createShuffleVectorInst(NewShuffle);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-void ShuffleVectorInst::setShuffleMask(ArrayRef<int> Mask) {
-  Ctx.getTracker().emplaceIfTracking<ShuffleVectorSetMask>(this);
-  cast<llvm::ShuffleVectorInst>(Val)->setShuffleMask(Mask);
-}
-
-VectorType *ShuffleVectorInst::getType() const {
-  return cast<VectorType>(
-      Ctx.getType(cast<llvm::ShuffleVectorInst>(Val)->getType()));
-}
-
-void ShuffleVectorInst::commute() {
-  Ctx.getTracker().emplaceIfTracking<ShuffleVectorSetMask>(this);
-  Ctx.getTracker().emplaceIfTracking<UseSwap>(getOperandUse(0),
-                                              getOperandUse(1));
-  cast<llvm::ShuffleVectorInst>(Val)->commute();
-}
-
-Constant *ShuffleVectorInst::getShuffleMaskForBitcode() const {
-  return Ctx.getOrCreateConstant(
-      cast<llvm::ShuffleVectorInst>(Val)->getShuffleMaskForBitcode());
-}
-
-Constant *ShuffleVectorInst::convertShuffleMaskForBitcode(ArrayRef<int> Mask,
-                                                          Type *ResultTy) {
-  return ResultTy->getContext().getOrCreateConstant(
-      llvm::ShuffleVectorInst::convertShuffleMaskForBitcode(Mask,
-                                                            ResultTy->LLVMTy));
-}
-
-VectorType *ExtractElementInst::getVectorOperandType() const {
-  return cast<VectorType>(Ctx.getType(getVectorOperand()->getType()->LLVMTy));
-}
-
-Value *ExtractValueInst::create(Value *Agg, ArrayRef<unsigned> Idxs,
-                                BBIterator WhereIt, BasicBlock *WhereBB,
-                                Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::Value *NewV = Builder.CreateExtractValue(Agg->Val, Idxs, Name);
-  if (auto *NewExtractValueInst = dyn_cast<llvm::ExtractValueInst>(NewV))
-    return Ctx.createExtractValueInst(NewExtractValueInst);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-Type *ExtractValueInst::getIndexedType(Type *Agg, ArrayRef<unsigned> Idxs) {
-  auto *LLVMTy = llvm::ExtractValueInst::getIndexedType(Agg->LLVMTy, Idxs);
-  return Agg->getContext().getType(LLVMTy);
-}
-
-Value *InsertValueInst::create(Value *Agg, Value *Val, ArrayRef<unsigned> Idxs,
-                               BBIterator WhereIt, BasicBlock *WhereBB,
-                               Context &Ctx, const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  if (WhereIt != WhereBB->end())
-    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
-  else
-    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
-  llvm::Value *NewV = Builder.CreateInsertValue(Agg->Val, Val->Val, Idxs, Name);
-  if (auto *NewInsertValueInst = dyn_cast<llvm::InsertValueInst>(NewV))
-    return Ctx.createInsertValueInst(NewInsertValueInst);
-  assert(isa<llvm::Constant>(NewV) && "Expected constant");
-  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
-}
-
-#ifndef NDEBUG
-void Constant::dumpOS(raw_ostream &OS) const {
-  dumpCommonPrefix(OS);
-  dumpCommonSuffix(OS);
-}
-#endif // NDEBUG
-
-ConstantInt *ConstantInt::getTrue(Context &Ctx) {
-  auto *LLVMC = llvm::ConstantInt::getTrue(Ctx.LLVMCtx);
-  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
-}
-ConstantInt *ConstantInt::getFalse(Context &Ctx) {
-  auto *LLVMC = llvm::ConstantInt::getFalse(Ctx.LLVMCtx);
-  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
-}
-ConstantInt *ConstantInt::getBool(Context &Ctx, bool V) {
-  auto *LLVMC = llvm::ConstantInt::getBool(Ctx.LLVMCtx, V);
-  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
-}
-Constant *ConstantInt::getTrue(Type *Ty) {
-  auto *LLVMC = llvm::ConstantInt::getTrue(Ty->LLVMTy);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-Constant *ConstantInt::getFalse(Type *Ty) {
-  auto *LLVMC = llvm::ConstantInt::getFalse(Ty->LLVMTy);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-Constant *ConstantInt::getBool(Type *Ty, bool V) {
-  auto *LLVMC = llvm::ConstantInt::getBool(Ty->LLVMTy, V);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-ConstantInt *ConstantInt::get(Type *Ty, uint64_t V, bool IsSigned) {
-  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V, IsSigned);
-  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool IsSigned) {
-  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V, IsSigned);
-  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-ConstantInt *ConstantInt::getSigned(IntegerType *Ty, int64_t V) {
-  auto *LLVMC =
-      llvm::ConstantInt::getSigned(cast<llvm::IntegerType>(Ty->LLVMTy), V);
-  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantInt::getSigned(Type *Ty, int64_t V) {
-  auto *LLVMC = llvm::ConstantInt::getSigned(Ty->LLVMTy, V);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-ConstantInt *ConstantInt::get(Context &Ctx, const APInt &V) {
-  auto *LLVMC = llvm::ConstantInt::get(Ctx.LLVMCtx, V);
-  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
-}
-ConstantInt *ConstantInt::get(IntegerType *Ty, StringRef Str, uint8_t Radix) {
-  auto *LLVMC =
-      llvm::ConstantInt::get(cast<llvm::IntegerType>(Ty->LLVMTy), Str, Radix);
-  return cast<ConstantInt>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantInt::get(Type *Ty, const APInt &V) {
-  auto *LLVMC = llvm::ConstantInt::get(Ty->LLVMTy, V);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-IntegerType *ConstantInt::getIntegerType() const {
-  auto *LLVMTy = cast<llvm::ConstantInt>(Val)->getIntegerType();
-  return cast<IntegerType>(Ctx.getType(LLVMTy));
-}
-
-bool ConstantInt::isValueValidForType(Type *Ty, uint64_t V) {
-  return llvm::ConstantInt::isValueValidForType(Ty->LLVMTy, V);
-}
-bool ConstantInt::isValueValidForType(Type *Ty, int64_t V) {
-  return llvm::ConstantInt::isValueValidForType(Ty->LLVMTy, V);
-}
-
-Constant *ConstantFP::get(Type *Ty, double V) {
-  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, V);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-
-Constant *ConstantFP::get(Type *Ty, const APFloat &V) {
-  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, V);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-
-Constant *ConstantFP::get(Type *Ty, StringRef Str) {
-  auto *LLVMC = llvm::ConstantFP::get(Ty->LLVMTy, Str);
-  return Ty->getContext().getOrCreateConstant(LLVMC);
-}
-
-ConstantFP *ConstantFP::get(const APFloat &V, Context &Ctx) {
-  auto *LLVMC = llvm::ConstantFP::get(Ctx.LLVMCtx, V);
-  return cast<ConstantFP>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-Constant *ConstantFP::getNaN(Type *Ty, bool Negative, uint64_t Payload) {
-  auto *LLVMC = llvm::ConstantFP::getNaN(Ty->LLVMTy, Negative, Payload);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantFP::getQNaN(Type *Ty, bool Negative, APInt *Payload) {
-  auto *LLVMC = llvm::ConstantFP::getQNaN(Ty->LLVMTy, Negative, Payload);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantFP::getSNaN(Type *Ty, bool Negative, APInt *Payload) {
-  auto *LLVMC = llvm::ConstantFP::getSNaN(Ty->LLVMTy, Negative, Payload);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantFP::getZero(Type *Ty, bool Negative) {
-  auto *LLVMC = llvm::ConstantFP::getZero(Ty->LLVMTy, Negative);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantFP::getNegativeZero(Type *Ty) {
-  auto *LLVMC = llvm::ConstantFP::getNegativeZero(Ty->LLVMTy);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) {
-  auto *LLVMC = llvm::ConstantFP::getInfinity(Ty->LLVMTy, Negative);
-  return cast<Constant>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-bool ConstantFP::isValueValidForType(Type *Ty, const APFloat &V) {
-  return llvm::ConstantFP::isValueValidForType(Ty->LLVMTy, V);
-}
-
-Constant *ConstantArray::get(ArrayType *T, ArrayRef<Constant *> V) {
-  auto &Ctx = T->getContext();
-  SmallVector<llvm::Constant *> LLVMValues;
-  LLVMValues.reserve(V.size());
-  for (auto *Elm : V)
-    LLVMValues.push_back(cast<llvm::Constant>(Elm->Val));
-  auto *LLVMC =
-      llvm::ConstantArray::get(cast<llvm::ArrayType>(T->LLVMTy), LLVMValues);
-  return cast<ConstantArray>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-ArrayType *ConstantArray::getType() const {
-  return cast<ArrayType>(
-      Ctx.getType(cast<llvm::ConstantArray>(Val)->getType()));
-}
-
-Constant *ConstantStruct::get(StructType *T, ArrayRef<Constant *> V) {
-  auto &Ctx = T->getContext();
-  SmallVector<llvm::Constant *> LLVMValues;
-  LLVMValues.reserve(V.size());
-  for (auto *Elm : V)
-    LLVMValues.push_back(cast<llvm::Constant>(Elm->Val));
-  auto *LLVMC =
-      llvm::ConstantStruct::get(cast<llvm::StructType>(T->LLVMTy), LLVMValues);
-  return cast<ConstantStruct>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-StructType *ConstantStruct::getTypeForElements(Context &Ctx,
-                                               ArrayRef<Constant *> V,
-                                               bool Packed) {
-  unsigned VecSize = V.size();
-  SmallVector<Type *, 16> EltTypes;
-  EltTypes.reserve(VecSize);
-  for (Constant *Elm : V)
-    EltTypes.push_back(Elm->getType());
-  return StructType::get(Ctx, EltTypes, Packed);
-}
-
-ConstantAggregateZero *ConstantAggregateZero::get(Type *Ty) {
-  auto *LLVMC = llvm::ConstantAggregateZero::get(Ty->LLVMTy);
-  return cast<ConstantAggregateZero>(
-      Ty->getContext().getOrCreateConstant(LLVMC));
-}
-
-Constant *ConstantAggregateZero::getSequentialElement() const {
-  return cast<Constant>(Ctx.getValue(
-      cast<llvm::ConstantAggregateZero>(Val)->getSequentialElement()));
-}
-Constant *ConstantAggregateZero::getStructElement(unsigned Elt) const {
-  return cast<Constant>(Ctx.getValue(
-      cast<llvm::ConstantAggregateZero>(Val)->getStructElement(Elt)));
-}
-Constant *ConstantAggregateZero::getElementValue(Constant *C) const {
-  return cast<Constant>(
-      Ctx.getValue(cast<llvm::ConstantAggregateZero>(Val)->getElementValue(
-          cast<llvm::Constant>(C->Val))));
-}
-Constant *ConstantAggregateZero::getElementValue(unsigned Idx) const {
-  return cast<Constant>(Ctx.getValue(
-      cast<llvm::ConstantAggregateZero>(Val)->getElementValue(Idx)));
-}
-
-ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) {
-  auto *LLVMC =
-      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(Ty->LLVMTy));
-  return cast<ConstantPointerNull>(Ty->getContext().getOrCreateConstant(LLVMC));
-}
-
-PointerType *ConstantPointerNull::getType() const {
-  return cast<PointerType>(
-      Ctx.getType(cast<llvm::ConstantPointerNull>(Val)->getType()));
-}
-
-UndefValue *UndefValue::get(Type *T) {
-  auto *LLVMC = llvm::UndefValue::get(T->LLVMTy);
-  return cast<UndefValue>(T->getContext().getOrCreateConstant(LLVMC));
-}
-
-UndefValue *UndefValue::getSequentialElement() const {
-  return cast<UndefValue>(Ctx.getOrCreateConstant(
-      cast<llvm::UndefValue>(Val)->getSequentialElement()));
-}
-
-UndefValue *UndefValue::getStructElement(unsigned Elt) const {
-  return cast<UndefValue>(Ctx.getOrCreateConstant(
-      cast<llvm::UndefValue>(Val)->getStructElement(Elt)));
-}
-
-UndefValue *UndefValue::getElementValue(Constant *C) const {
-  return cast<UndefValue>(
-      Ctx.getOrCreateConstant(cast<llvm::UndefValue>(Val)->getElementValue(
-          cast<llvm::Constant>(C->Val))));
-}
-
-UndefValue *UndefValue::getElementValue(unsigned Idx) const {
-  return cast<UndefValue>(Ctx.getOrCreateConstant(
-      cast<llvm::UndefValue>(Val)->getElementValue(Idx)));
-}
-
-PoisonValue *PoisonValue::get(Type *T) {
-  auto *LLVMC = llvm::PoisonValue::get(T->LLVMTy);
-  return cast<PoisonValue>(T->getContext().getOrCreateConstant(LLVMC));
-}
-
-PoisonValue *PoisonValue::getSequentialElement() const {
-  return cast<PoisonValue>(Ctx.getOrCreateConstant(
-      cast<llvm::PoisonValue>(Val)->getSequentialElement()));
-}
-
-PoisonValue *PoisonValue::getStructElement(unsigned Elt) const {
-  return cast<PoisonValue>(Ctx.getOrCreateConstant(
-      cast<llvm::PoisonValue>(Val)->getStructElement(Elt)));
-}
-
-PoisonValue *PoisonValue::getElementValue(Constant *C) const {
-  return cast<PoisonValue>(
-      Ctx.getOrCreateConstant(cast<llvm::PoisonValue>(Val)->getElementValue(
-          cast<llvm::Constant>(C->Val))));
-}
-
-PoisonValue *PoisonValue::getElementValue(unsigned Idx) const {
-  return cast<PoisonValue>(Ctx.getOrCreateConstant(
-      cast<llvm::PoisonValue>(Val)->getElementValue(Idx)));
-}
-
-void GlobalObject::setAlignment(MaybeAlign Align) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalObject::getAlign, &GlobalObject::setAlignment>>(
-          this);
-  cast<llvm::GlobalObject>(Val)->setAlignment(Align);
-}
-
-void GlobalObject::setGlobalObjectSubClassData(unsigned V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalObject::getGlobalObjectSubClassData,
-                        &GlobalObject::setGlobalObjectSubClassData>>(this);
-  cast<llvm::GlobalObject>(Val)->setGlobalObjectSubClassData(V);
-}
-
-void GlobalObject::setSection(StringRef S) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalObject::getSection, &GlobalObject::setSection>>(
-          this);
-  cast<llvm::GlobalObject>(Val)->setSection(S);
-}
-
-template <typename GlobalT, typename LLVMGlobalT, typename ParentT,
-          typename LLVMParentT>
-GlobalT &GlobalWithNodeAPI<GlobalT, LLVMGlobalT, ParentT, LLVMParentT>::
-    LLVMGVToGV::operator()(LLVMGlobalT &LLVMGV) const {
-  return cast<GlobalT>(*Ctx.getValue(&LLVMGV));
-}
-
-namespace llvm::sandboxir {
-// Explicit instantiations.
-template class GlobalWithNodeAPI<GlobalIFunc, llvm::GlobalIFunc, GlobalObject,
-                                 llvm::GlobalObject>;
-template class GlobalWithNodeAPI<Function, llvm::Function, GlobalObject,
-                                 llvm::GlobalObject>;
-template class GlobalWithNodeAPI<GlobalVariable, llvm::GlobalVariable,
-                                 GlobalObject, llvm::GlobalObject>;
-template class GlobalWithNodeAPI<GlobalAlias, llvm::GlobalAlias, GlobalValue,
-                                 llvm::GlobalValue>;
-} // namespace llvm::sandboxir
-
-void GlobalIFunc::setResolver(Constant *Resolver) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalIFunc::getResolver, &GlobalIFunc::setResolver>>(
-          this);
-  cast<llvm::GlobalIFunc>(Val)->setResolver(
-      cast<llvm::Constant>(Resolver->Val));
-}
-
-Constant *GlobalIFunc::getResolver() const {
-  return Ctx.getOrCreateConstant(cast<llvm::GlobalIFunc>(Val)->getResolver());
-}
-
-Function *GlobalIFunc::getResolverFunction() {
-  return cast<Function>(Ctx.getOrCreateConstant(
-      cast<llvm::GlobalIFunc>(Val)->getResolverFunction()));
-}
-
-GlobalVariable &
-GlobalVariable::LLVMGVToGV::operator()(llvm::GlobalVariable &LLVMGV) const {
-  return cast<GlobalVariable>(*Ctx.getValue(&LLVMGV));
-}
-
-Constant *GlobalVariable::getInitializer() const {
-  return Ctx.getOrCreateConstant(
-      cast<llvm::GlobalVariable>(Val)->getInitializer());
-}
-
-void GlobalVariable::setInitializer(Constant *InitVal) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&GlobalVariable::getInitializer,
-                                       &GlobalVariable::setInitializer>>(this);
-  cast<llvm::GlobalVariable>(Val)->setInitializer(
-      cast<llvm::Constant>(InitVal->Val));
-}
-
-void GlobalVariable::setConstant(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&GlobalVariable::isConstant,
-                                       &GlobalVariable::setConstant>>(this);
-  cast<llvm::GlobalVariable>(Val)->setConstant(V);
-}
-
-void GlobalVariable::setExternallyInitialized(bool V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalVariable::isExternallyInitialized,
-                        &GlobalVariable::setExternallyInitialized>>(this);
-  cast<llvm::GlobalVariable>(Val)->setExternallyInitialized(V);
-}
-
-void GlobalAlias::setAliasee(Constant *Aliasee) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&GlobalAlias::getAliasee, &GlobalAlias::setAliasee>>(
-          this);
-  cast<llvm::GlobalAlias>(Val)->setAliasee(cast<llvm::Constant>(Aliasee->Val));
-}
-
-Constant *GlobalAlias::getAliasee() const {
-  return cast<Constant>(
-      Ctx.getOrCreateConstant(cast<llvm::GlobalAlias>(Val)->getAliasee()));
-}
-
-const GlobalObject *GlobalAlias::getAliaseeObject() const {
-  return cast<GlobalObject>(Ctx.getOrCreateConstant(
-      cast<llvm::GlobalAlias>(Val)->getAliaseeObject()));
-}
-
-void GlobalValue::setUnnamedAddr(UnnamedAddr V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&GlobalValue::getUnnamedAddr,
-                                       &GlobalValue::setUnnamedAddr>>(this);
-  cast<llvm::GlobalValue>(Val)->setUnnamedAddr(V);
-}
-
-void GlobalValue::setVisibility(VisibilityTypes V) {
-  Ctx.getTracker()
-      .emplaceIfTracking<GenericSetter<&GlobalValue::getVisibility,
-                                       &GlobalValue::setVisibility>>(this);
-  cast<llvm::GlobalValue>(Val)->setVisibility(V);
-}
-
-NoCFIValue *NoCFIValue::get(GlobalValue *GV) {
-  auto *LLVMC = llvm::NoCFIValue::get(cast<llvm::GlobalValue>(GV->Val));
-  return cast<NoCFIValue>(GV->getContext().getOrCreateConstant(LLVMC));
-}
-
-GlobalValue *NoCFIValue::getGlobalValue() const {
-  auto *LLVMC = cast<llvm::NoCFIValue>(Val)->getGlobalValue();
-  return cast<GlobalValue>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-PointerType *NoCFIValue::getType() const {
-  return cast<PointerType>(Ctx.getType(cast<llvm::NoCFIValue>(Val)->getType()));
-}
-
-ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key,
-                                      ConstantInt *Disc, Constant *AddrDisc) {
-  auto *LLVMC = llvm::ConstantPtrAuth::get(
-      cast<llvm::Constant>(Ptr->Val), cast<llvm::ConstantInt>(Key->Val),
-      cast<llvm::ConstantInt>(Disc->Val), cast<llvm::Constant>(AddrDisc->Val));
-  return cast<ConstantPtrAuth>(Ptr->getContext().getOrCreateConstant(LLVMC));
-}
-
-Constant *ConstantPtrAuth::getPointer() const {
-  return Ctx.getOrCreateConstant(
-      cast<llvm::ConstantPtrAuth>(Val)->getPointer());
-}
-
-ConstantInt *ConstantPtrAuth::getKey() const {
-  return cast<ConstantInt>(
-      Ctx.getOrCreateConstant(cast<llvm::ConstantPtrAuth>(Val)->getKey()));
-}
-
-ConstantInt *ConstantPtrAuth::getDiscriminator() const {
-  return cast<ConstantInt>(Ctx.getOrCreateConstant(
-      cast<llvm::ConstantPtrAuth>(Val)->getDiscriminator()));
-}
-
-Constant *ConstantPtrAuth::getAddrDiscriminator() const {
-  return Ctx.getOrCreateConstant(
-      cast<llvm::ConstantPtrAuth>(Val)->getAddrDiscriminator());
-}
-
-ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const {
-  auto *LLVMC = cast<llvm::ConstantPtrAuth>(Val)->getWithSameSchema(
-      cast<llvm::Constant>(Pointer->Val));
-  return cast<ConstantPtrAuth>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-BlockAddress *BlockAddress::get(Function *F, BasicBlock *BB) {
-  auto *LLVMC = llvm::BlockAddress::get(cast<llvm::Function>(F->Val),
-                                        cast<llvm::BasicBlock>(BB->Val));
-  return cast<BlockAddress>(F->getContext().getOrCreateConstant(LLVMC));
-}
-
-BlockAddress *BlockAddress::get(BasicBlock *BB) {
-  auto *LLVMC = llvm::BlockAddress::get(cast<llvm::BasicBlock>(BB->Val));
-  return cast<BlockAddress>(BB->getContext().getOrCreateConstant(LLVMC));
-}
-
-BlockAddress *BlockAddress::lookup(const BasicBlock *BB) {
-  auto *LLVMC = llvm::BlockAddress::lookup(cast<llvm::BasicBlock>(BB->Val));
-  return cast_or_null<BlockAddress>(BB->getContext().getValue(LLVMC));
-}
-
-Function *BlockAddress::getFunction() const {
-  return cast<Function>(
-      Ctx.getValue(cast<llvm::BlockAddress>(Val)->getFunction()));
-}
-
-BasicBlock *BlockAddress::getBasicBlock() const {
-  return cast<BasicBlock>(
-      Ctx.getValue(cast<llvm::BlockAddress>(Val)->getBasicBlock()));
-}
-
-DSOLocalEquivalent *DSOLocalEquivalent::get(GlobalValue *GV) {
-  auto *LLVMC = llvm::DSOLocalEquivalent::get(cast<llvm::GlobalValue>(GV->Val));
-  return cast<DSOLocalEquivalent>(GV->getContext().getValue(LLVMC));
-}
-
-GlobalValue *DSOLocalEquivalent::getGlobalValue() const {
-  return cast<GlobalValue>(
-      Ctx.getValue(cast<llvm::DSOLocalEquivalent>(Val)->getGlobalValue()));
-}
-
-ConstantTokenNone *ConstantTokenNone::get(Context &Ctx) {
-  auto *LLVMC = llvm::ConstantTokenNone::get(Ctx.LLVMCtx);
-  return cast<ConstantTokenNone>(Ctx.getOrCreateConstant(LLVMC));
-}
-
-FunctionType *Function::getFunctionType() const {
-  return cast<FunctionType>(
-      Ctx.getType(cast<llvm::Function>(Val)->getFunctionType()));
-}
-
-#ifndef NDEBUG
-void Function::dumpNameAndArgs(raw_ostream &OS) const {
-  auto *F = cast<llvm::Function>(Val);
-  OS << *F->getReturnType() << " @" << F->getName() << "(";
-  interleave(
-      F->args(),
-      [this, &OS](const llvm::Argument &LLVMArg) {
-        auto *SBArg = cast_or_null<Argument>(Ctx.getValue(&LLVMArg));
-        if (SBArg == nullptr)
-          OS << "NULL";
-        else
-          SBArg->printAsOperand(OS);
-      },
-      [&] { OS << ", "; });
-  OS << ")";
-}
-void Function::dumpOS(raw_ostream &OS) const {
-  dumpNameAndArgs(OS);
-  OS << " {\n";
-  auto *LLVMF = cast<llvm::Function>(Val);
-  interleave(
-      *LLVMF,
-      [this, &OS](const llvm::BasicBlock &LLVMBB) {
-        auto *BB = cast_or_null<BasicBlock>(Ctx.getValue(&LLVMBB));
-        if (BB == nullptr)
-          OS << "NULL";
-        else
-          OS << *BB;
-      },
-      [&OS] { OS << "\n"; });
-  OS << "}\n";
-}
-#endif // NDEBUG
-
-BasicBlock::iterator::pointer
-BasicBlock::iterator::getInstr(llvm::BasicBlock::iterator It) const {
-  return cast_or_null<Instruction>(Ctx->getValue(&*It));
-}
-
-std::unique_ptr<Value> Context::detachLLVMValue(llvm::Value *V) {
-  std::unique_ptr<Value> Erased;
-  auto It = LLVMValueToValueMap.find(V);
-  if (It != LLVMValueToValueMap.end()) {
-    auto *Val = It->second.release();
-    Erased = std::unique_ptr<Value>(Val);
-    LLVMValueToValueMap.erase(It);
-  }
-  return Erased;
-}
-
-std::unique_ptr<Value> Context::detach(Value *V) {
-  assert(V->getSubclassID() != Value::ClassID::Constant &&
-         "Can't detach a constant!");
-  assert(V->getSubclassID() != Value::ClassID::User && "Can't detach a user!");
-  return detachLLVMValue(V->Val);
-}
-
-Value *Context::registerValue(std::unique_ptr<Value> &&VPtr) {
-  assert(VPtr->getSubclassID() != Value::ClassID::User &&
-         "Can't register a user!");
-
-  // Track creation of instructions.
-  // Please note that we don't allow the creation of detached instructions,
-  // meaning that the instructions need to be inserted into a block upon
-  // creation. This is why the tracker class combines creation and insertion.
-  if (auto *I = dyn_cast<Instruction>(VPtr.get()))
-    getTracker().emplaceIfTracking<CreateAndInsertInst>(I);
-
-  Value *V = VPtr.get();
-  [[maybe_unused]] auto Pair =
-      LLVMValueToValueMap.insert({VPtr->Val, std::move(VPtr)});
-  assert(Pair.second && "Already exists!");
-  return V;
-}
-
-Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
-  auto Pair = LLVMValueToValueMap.insert({LLVMV, nullptr});
-  auto It = Pair.first;
-  if (!Pair.second)
-    return It->second.get();
-
-  if (auto *C = dyn_cast<llvm::Constant>(LLVMV)) {
-    switch (C->getValueID()) {
-    case llvm::Value::ConstantIntVal:
-      It->second = std::unique_ptr<ConstantInt>(
-          new ConstantInt(cast<llvm::ConstantInt>(C), *this));
-      return It->second.get();
-    case llvm::Value::ConstantFPVal:
-      It->second = std::unique_ptr<ConstantFP>(
-          new ConstantFP(cast<llvm::ConstantFP>(C), *this));
-      return It->second.get();
-    case llvm::Value::BlockAddressVal:
-      It->second = std::unique_ptr<BlockAddress>(
-          new BlockAddress(cast<llvm::BlockAddress>(C), *this));
-      return It->second.get();
-    case llvm::Value::ConstantTokenNoneVal:
-      It->second = std::unique_ptr<ConstantTokenNone>(
-          new ConstantTokenNone(cast<llvm::ConstantTokenNone>(C), *this));
-      return It->second.get();
-    case llvm::Value::ConstantAggregateZeroVal: {
-      auto *CAZ = cast<llvm::ConstantAggregateZero>(C);
-      It->second = std::unique_ptr<ConstantAggregateZero>(
-          new ConstantAggregateZero(CAZ, *this));
-      auto *Ret = It->second.get();
-      // Must create sandboxir for elements.
-      auto EC = CAZ->getElementCount();
-      if (EC.isFixed()) {
-        for (auto ElmIdx : seq<unsigned>(0, EC.getFixedValue()))
-          getOrCreateValueInternal(CAZ->getElementValue(ElmIdx), CAZ);
-      }
-      return Ret;
-    }
-    case llvm::Value::ConstantPointerNullVal:
-      It->second = std::unique_ptr<ConstantPointerNull>(
-          new ConstantPointerNull(cast<llvm::ConstantPointerNull>(C), *this));
-      return It->second.get();
-    case llvm::Value::PoisonValueVal:
-      It->second = std::unique_ptr<PoisonValue>(
-          new PoisonValue(cast<llvm::PoisonValue>(C), *this));
-      return It->second.get();
-    case llvm::Value::UndefValueVal:
-      It->second = std::unique_ptr<UndefValue>(
-          new UndefValue(cast<llvm::UndefValue>(C), *this));
-      return It->second.get();
-    case llvm::Value::DSOLocalEquivalentVal: {
-      auto *DSOLE = cast<llvm::DSOLocalEquivalent>(C);
-      It->second = std::unique_ptr<DSOLocalEquivalent>(
-          new DSOLocalEquivalent(DSOLE, *this));
-      auto *Ret = It->second.get();
-      getOrCreateValueInternal(DSOLE->getGlobalValue(), DSOLE);
-      return Ret;
-    }
-    case llvm::Value::ConstantArrayVal:
-      It->second = std::unique_ptr<ConstantArray>(
-          new ConstantArray(cast<llvm::ConstantArray>(C), *this));
-      break;
-    case llvm::Value::ConstantStructVal:
-      It->second = std::unique_ptr<ConstantStruct>(
-          new ConstantStruct(cast<llvm::ConstantStruct>(C), *this));
-      break;
-    case llvm::Value::ConstantVectorVal:
-      It->second = std::unique_ptr<ConstantVector>(
-          new ConstantVector(cast<llvm::ConstantVector>(C), *this));
-      break;
-    case llvm::Value::FunctionVal:
-      It->second = std::unique_ptr<Function>(
-          new Function(cast<llvm::Function>(C), *this));
-      break;
-    case llvm::Value::GlobalIFuncVal:
-      It->second = std::unique_ptr<GlobalIFunc>(
-          new GlobalIFunc(cast<llvm::GlobalIFunc>(C), *this));
-      break;
-    case llvm::Value::GlobalVariableVal:
-      It->second = std::unique_ptr<GlobalVariable>(
-          new GlobalVariable(cast<llvm::GlobalVariable>(C), *this));
-      break;
-    case llvm::Value::GlobalAliasVal:
-      It->second = std::unique_ptr<GlobalAlias>(
-          new GlobalAlias(cast<llvm::GlobalAlias>(C), *this));
-      break;
-    case llvm::Value::NoCFIValueVal:
-      It->second = std::unique_ptr<NoCFIValue>(
-          new NoCFIValue(cast<llvm::NoCFIValue>(C), *this));
-      break;
-    case llvm::Value::ConstantPtrAuthVal:
-      It->second = std::unique_ptr<ConstantPtrAuth>(
-          new ConstantPtrAuth(cast<llvm::ConstantPtrAuth>(C), *this));
-      break;
-    case llvm::Value::ConstantExprVal:
-      It->second = std::unique_ptr<ConstantExpr>(
-          new ConstantExpr(cast<llvm::ConstantExpr>(C), *this));
-      break;
-    default:
-      It->second = std::unique_ptr<Constant>(new Constant(C, *this));
-      break;
-    }
-    auto *NewC = It->second.get();
-    for (llvm::Value *COp : C->operands())
-      getOrCreateValueInternal(COp, C);
-    return NewC;
-  }
-  if (auto *Arg = dyn_cast<llvm::Argument>(LLVMV)) {
-    It->second = std::unique_ptr<Argument>(new Argument(Arg, *this));
-    return It->second.get();
-  }
-  if (auto *BB = dyn_cast<llvm::BasicBlock>(LLVMV)) {
-    assert(isa<llvm::BlockAddress>(U) &&
-           "This won't create a SBBB, don't call this function directly!");
-    if (auto *SBBB = getValue(BB))
-      return SBBB;
-    return nullptr;
-  }
-  assert(isa<llvm::Instruction>(LLVMV) && "Expected Instruction");
-
-  switch (cast<llvm::Instruction>(LLVMV)->getOpcode()) {
-  case llvm::Instruction::VAArg: {
-    auto *LLVMVAArg = cast<llvm::VAArgInst>(LLVMV);
-    It->second = std::unique_ptr<VAArgInst>(new VAArgInst(LLVMVAArg, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Freeze: {
-    auto *LLVMFreeze = cast<llvm::FreezeInst>(LLVMV);
-    It->second = std::unique_ptr<FreezeInst>(new FreezeInst(LLVMFreeze, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Fence: {
-    auto *LLVMFence = cast<llvm::FenceInst>(LLVMV);
-    It->second = std::unique_ptr<FenceInst>(new FenceInst(LLVMFence, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Select: {
-    auto *LLVMSel = cast<llvm::SelectInst>(LLVMV);
-    It->second = std::unique_ptr<SelectInst>(new SelectInst(LLVMSel, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::ExtractElement: {
-    auto *LLVMIns = cast<llvm::ExtractElementInst>(LLVMV);
-    It->second = std::unique_ptr<ExtractElementInst>(
-        new ExtractElementInst(LLVMIns, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::InsertElement: {
-    auto *LLVMIns = cast<llvm::InsertElementInst>(LLVMV);
-    It->second = std::unique_ptr<InsertElementInst>(
-        new InsertElementInst(LLVMIns, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::ShuffleVector: {
-    auto *LLVMIns = cast<llvm::ShuffleVectorInst>(LLVMV);
-    It->second = std::unique_ptr<ShuffleVectorInst>(
-        new ShuffleVectorInst(LLVMIns, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::ExtractValue: {
-    auto *LLVMIns = cast<llvm::ExtractValueInst>(LLVMV);
-    It->second =
-        std::unique_ptr<ExtractValueInst>(new ExtractValueInst(LLVMIns, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::InsertValue: {
-    auto *LLVMIns = cast<llvm::InsertValueInst>(LLVMV);
-    It->second =
-        std::unique_ptr<InsertValueInst>(new InsertValueInst(LLVMIns, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Br: {
-    auto *LLVMBr = cast<llvm::BranchInst>(LLVMV);
-    It->second = std::unique_ptr<BranchInst>(new BranchInst(LLVMBr, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Load: {
-    auto *LLVMLd = cast<llvm::LoadInst>(LLVMV);
-    It->second = std::unique_ptr<LoadInst>(new LoadInst(LLVMLd, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Store: {
-    auto *LLVMSt = cast<llvm::StoreInst>(LLVMV);
-    It->second = std::unique_ptr<StoreInst>(new StoreInst(LLVMSt, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Ret: {
-    auto *LLVMRet = cast<llvm::ReturnInst>(LLVMV);
-    It->second = std::unique_ptr<ReturnInst>(new ReturnInst(LLVMRet, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Call: {
-    auto *LLVMCall = cast<llvm::CallInst>(LLVMV);
-    It->second = std::unique_ptr<CallInst>(new CallInst(LLVMCall, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Invoke: {
-    auto *LLVMInvoke = cast<llvm::InvokeInst>(LLVMV);
-    It->second = std::unique_ptr<InvokeInst>(new InvokeInst(LLVMInvoke, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CallBr: {
-    auto *LLVMCallBr = cast<llvm::CallBrInst>(LLVMV);
-    It->second = std::unique_ptr<CallBrInst>(new CallBrInst(LLVMCallBr, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::LandingPad: {
-    auto *LLVMLPad = cast<llvm::LandingPadInst>(LLVMV);
-    It->second =
-        std::unique_ptr<LandingPadInst>(new LandingPadInst(LLVMLPad, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CatchPad: {
-    auto *LLVMCPI = cast<llvm::CatchPadInst>(LLVMV);
-    It->second =
-        std::unique_ptr<CatchPadInst>(new CatchPadInst(LLVMCPI, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CleanupPad: {
-    auto *LLVMCPI = cast<llvm::CleanupPadInst>(LLVMV);
-    It->second =
-        std::unique_ptr<CleanupPadInst>(new CleanupPadInst(LLVMCPI, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CatchRet: {
-    auto *LLVMCRI = cast<llvm::CatchReturnInst>(LLVMV);
-    It->second =
-        std::unique_ptr<CatchReturnInst>(new CatchReturnInst(LLVMCRI, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CleanupRet: {
-    auto *LLVMCRI = cast<llvm::CleanupReturnInst>(LLVMV);
-    It->second = std::unique_ptr<CleanupReturnInst>(
-        new CleanupReturnInst(LLVMCRI, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::GetElementPtr: {
-    auto *LLVMGEP = cast<llvm::GetElementPtrInst>(LLVMV);
-    It->second = std::unique_ptr<GetElementPtrInst>(
-        new GetElementPtrInst(LLVMGEP, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::CatchSwitch: {
-    auto *LLVMCatchSwitchInst = cast<llvm::CatchSwitchInst>(LLVMV);
-    It->second = std::unique_ptr<CatchSwitchInst>(
-        new CatchSwitchInst(LLVMCatchSwitchInst, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Resume: {
-    auto *LLVMResumeInst = cast<llvm::ResumeInst>(LLVMV);
-    It->second =
-        std::unique_ptr<ResumeInst>(new ResumeInst(LLVMResumeInst, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Switch: {
-    auto *LLVMSwitchInst = cast<llvm::SwitchInst>(LLVMV);
-    It->second =
-        std::unique_ptr<SwitchInst>(new SwitchInst(LLVMSwitchInst, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::FNeg: {
-    auto *LLVMUnaryOperator = cast<llvm::UnaryOperator>(LLVMV);
-    It->second = std::unique_ptr<UnaryOperator>(
-        new UnaryOperator(LLVMUnaryOperator, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Add:
-  case llvm::Instruction::FAdd:
-  case llvm::Instruction::Sub:
-  case llvm::Instruction::FSub:
-  case llvm::Instruction::Mul:
-  case llvm::Instruction::FMul:
-  case llvm::Instruction::UDiv:
-  case llvm::Instruction::SDiv:
-  case llvm::Instruction::FDiv:
-  case llvm::Instruction::URem:
-  case llvm::Instruction::SRem:
-  case llvm::Instruction::FRem:
-  case llvm::Instruction::Shl:
-  case llvm::Instruction::LShr:
-  case llvm::Instruction::AShr:
-  case llvm::Instruction::And:
-  case llvm::Instruction::Or:
-  case llvm::Instruction::Xor: {
-    auto *LLVMBinaryOperator = cast<llvm::BinaryOperator>(LLVMV);
-    It->second = std::unique_ptr<BinaryOperator>(
-        new BinaryOperator(LLVMBinaryOperator, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::AtomicRMW: {
-    auto *LLVMAtomicRMW = cast<llvm::AtomicRMWInst>(LLVMV);
-    It->second =
-        std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(LLVMAtomicRMW, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::AtomicCmpXchg: {
-    auto *LLVMAtomicCmpXchg = cast<llvm::AtomicCmpXchgInst>(LLVMV);
-    It->second = std::unique_ptr<AtomicCmpXchgInst>(
-        new AtomicCmpXchgInst(LLVMAtomicCmpXchg, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Alloca: {
-    auto *LLVMAlloca = cast<llvm::AllocaInst>(LLVMV);
-    It->second = std::unique_ptr<AllocaInst>(new AllocaInst(LLVMAlloca, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::ZExt:
-  case llvm::Instruction::SExt:
-  case llvm::Instruction::FPToUI:
-  case llvm::Instruction::FPToSI:
-  case llvm::Instruction::FPExt:
-  case llvm::Instruction::PtrToInt:
-  case llvm::Instruction::IntToPtr:
-  case llvm::Instruction::SIToFP:
-  case llvm::Instruction::UIToFP:
-  case llvm::Instruction::Trunc:
-  case llvm::Instruction::FPTrunc:
-  case llvm::Instruction::BitCast:
-  case llvm::Instruction::AddrSpaceCast: {
-    auto *LLVMCast = cast<llvm::CastInst>(LLVMV);
-    It->second = std::unique_ptr<CastInst>(new CastInst(LLVMCast, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::PHI: {
-    auto *LLVMPhi = cast<llvm::PHINode>(LLVMV);
-    It->second = std::unique_ptr<PHINode>(new PHINode(LLVMPhi, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::ICmp: {
-    auto *LLVMICmp = cast<llvm::ICmpInst>(LLVMV);
-    It->second = std::unique_ptr<ICmpInst>(new ICmpInst(LLVMICmp, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::FCmp: {
-    auto *LLVMFCmp = cast<llvm::FCmpInst>(LLVMV);
-    It->second = std::unique_ptr<FCmpInst>(new FCmpInst(LLVMFCmp, *this));
-    return It->second.get();
-  }
-  case llvm::Instruction::Unreachable: {
-    auto *LLVMUnreachable = cast<llvm::UnreachableInst>(LLVMV);
-    It->second = std::unique_ptr<UnreachableInst>(
-        new UnreachableInst(LLVMUnreachable, *this));
-    return It->second.get();
-  }
-  default:
-    break;
-  }
-
-  It->second = std::unique_ptr<OpaqueInst>(
-      new OpaqueInst(cast<llvm::Instruction>(LLVMV), *this));
-  return It->second.get();
-}
-
-BasicBlock *Context::createBasicBlock(llvm::BasicBlock *LLVMBB) {
-  assert(getValue(LLVMBB) == nullptr && "Already exists!");
-  auto NewBBPtr = std::unique_ptr<BasicBlock>(new BasicBlock(LLVMBB, *this));
-  auto *BB = cast<BasicBlock>(registerValue(std::move(NewBBPtr)));
-  // Create SandboxIR for BB's body.
-  BB->buildBasicBlockFromLLVMIR(LLVMBB);
-  return BB;
-}
-
-VAArgInst *Context::createVAArgInst(llvm::VAArgInst *SI) {
-  auto NewPtr = std::unique_ptr<VAArgInst>(new VAArgInst(SI, *this));
-  return cast<VAArgInst>(registerValue(std::move(NewPtr)));
-}
-
-FreezeInst *Context::createFreezeInst(llvm::FreezeInst *SI) {
-  auto NewPtr = std::unique_ptr<FreezeInst>(new FreezeInst(SI, *this));
-  return cast<FreezeInst>(registerValue(std::move(NewPtr)));
-}
-
-FenceInst *Context::createFenceInst(llvm::FenceInst *SI) {
-  auto NewPtr = std::unique_ptr<FenceInst>(new FenceInst(SI, *this));
-  return cast<FenceInst>(registerValue(std::move(NewPtr)));
-}
-
-SelectInst *Context::createSelectInst(llvm::SelectInst *SI) {
-  auto NewPtr = std::unique_ptr<SelectInst>(new SelectInst(SI, *this));
-  return cast<SelectInst>(registerValue(std::move(NewPtr)));
-}
-
-ExtractElementInst *
-Context::createExtractElementInst(llvm::ExtractElementInst *EEI) {
-  auto NewPtr =
-      std::unique_ptr<ExtractElementInst>(new ExtractElementInst(EEI, *this));
-  return cast<ExtractElementInst>(registerValue(std::move(NewPtr)));
-}
-
-InsertElementInst *
-Context::createInsertElementInst(llvm::InsertElementInst *IEI) {
-  auto NewPtr =
-      std::unique_ptr<InsertElementInst>(new InsertElementInst(IEI, *this));
-  return cast<InsertElementInst>(registerValue(std::move(NewPtr)));
-}
-
-ShuffleVectorInst *
-Context::createShuffleVectorInst(llvm::ShuffleVectorInst *SVI) {
-  auto NewPtr =
-      std::unique_ptr<ShuffleVectorInst>(new ShuffleVectorInst(SVI, *this));
-  return cast<ShuffleVectorInst>(registerValue(std::move(NewPtr)));
-}
-
-ExtractValueInst *Context::createExtractValueInst(llvm::ExtractValueInst *EVI) {
-  auto NewPtr =
-      std::unique_ptr<ExtractValueInst>(new ExtractValueInst(EVI, *this));
-  return cast<ExtractValueInst>(registerValue(std::move(NewPtr)));
-}
-
-InsertValueInst *Context::createInsertValueInst(llvm::InsertValueInst *IVI) {
-  auto NewPtr =
-      std::unique_ptr<InsertValueInst>(new InsertValueInst(IVI, *this));
-  return cast<InsertValueInst>(registerValue(std::move(NewPtr)));
-}
-
-BranchInst *Context::createBranchInst(llvm::BranchInst *BI) {
-  auto NewPtr = std::unique_ptr<BranchInst>(new BranchInst(BI, *this));
-  return cast<BranchInst>(registerValue(std::move(NewPtr)));
-}
-
-LoadInst *Context::createLoadInst(llvm::LoadInst *LI) {
-  auto NewPtr = std::unique_ptr<LoadInst>(new LoadInst(LI, *this));
-  return cast<LoadInst>(registerValue(std::move(NewPtr)));
-}
-
-StoreInst *Context::createStoreInst(llvm::StoreInst *SI) {
-  auto NewPtr = std::unique_ptr<StoreInst>(new StoreInst(SI, *this));
-  return cast<StoreInst>(registerValue(std::move(NewPtr)));
-}
-
-ReturnInst *Context::createReturnInst(llvm::ReturnInst *I) {
-  auto NewPtr = std::unique_ptr<ReturnInst>(new ReturnInst(I, *this));
-  return cast<ReturnInst>(registerValue(std::move(NewPtr)));
-}
-
-CallInst *Context::createCallInst(llvm::CallInst *I) {
-  auto NewPtr = std::unique_ptr<CallInst>(new CallInst(I, *this));
-  return cast<CallInst>(registerValue(std::move(NewPtr)));
-}
-
-InvokeInst *Context::createInvokeInst(llvm::InvokeInst *I) {
-  auto NewPtr = std::unique_ptr<InvokeInst>(new InvokeInst(I, *this));
-  return cast<InvokeInst>(registerValue(std::move(NewPtr)));
-}
-
-CallBrInst *Context::createCallBrInst(llvm::CallBrInst *I) {
-  auto NewPtr = std::unique_ptr<CallBrInst>(new CallBrInst(I, *this));
-  return cast<CallBrInst>(registerValue(std::move(NewPtr)));
-}
-
-UnreachableInst *Context::createUnreachableInst(llvm::UnreachableInst *UI) {
-  auto NewPtr =
-      std::unique_ptr<UnreachableInst>(new UnreachableInst(UI, *this));
-  return cast<UnreachableInst>(registerValue(std::move(NewPtr)));
-}
-LandingPadInst *Context::createLandingPadInst(llvm::LandingPadInst *I) {
-  auto NewPtr = std::unique_ptr<LandingPadInst>(new LandingPadInst(I, *this));
-  return cast<LandingPadInst>(registerValue(std::move(NewPtr)));
-}
-CatchPadInst *Context::createCatchPadInst(llvm::CatchPadInst *I) {
-  auto NewPtr = std::unique_ptr<CatchPadInst>(new CatchPadInst(I, *this));
-  return cast<CatchPadInst>(registerValue(std::move(NewPtr)));
-}
-CleanupPadInst *Context::createCleanupPadInst(llvm::CleanupPadInst *I) {
-  auto NewPtr = std::unique_ptr<CleanupPadInst>(new CleanupPadInst(I, *this));
-  return cast<CleanupPadInst>(registerValue(std::move(NewPtr)));
-}
-CatchReturnInst *Context::createCatchReturnInst(llvm::CatchReturnInst *I) {
-  auto NewPtr = std::unique_ptr<CatchReturnInst>(new CatchReturnInst(I, *this));
-  return cast<CatchReturnInst>(registerValue(std::move(NewPtr)));
-}
-CleanupReturnInst *
-Context::createCleanupReturnInst(llvm::CleanupReturnInst *I) {
-  auto NewPtr =
-      std::unique_ptr<CleanupReturnInst>(new CleanupReturnInst(I, *this));
-  return cast<CleanupReturnInst>(registerValue(std::move(NewPtr)));
-}
-GetElementPtrInst *
-Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
-  auto NewPtr =
-      std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
-  return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
-}
-CatchSwitchInst *Context::createCatchSwitchInst(llvm::CatchSwitchInst *I) {
-  auto NewPtr = std::unique_ptr<CatchSwitchInst>(new CatchSwitchInst(I, *this));
-  return cast<CatchSwitchInst>(registerValue(std::move(NewPtr)));
-}
-ResumeInst *Context::createResumeInst(llvm::ResumeInst *I) {
-  auto NewPtr = std::unique_ptr<ResumeInst>(new ResumeInst(I, *this));
-  return cast<ResumeInst>(registerValue(std::move(NewPtr)));
-}
-SwitchInst *Context::createSwitchInst(llvm::SwitchInst *I) {
-  auto NewPtr = std::unique_ptr<SwitchInst>(new SwitchInst(I, *this));
-  return cast<SwitchInst>(registerValue(std::move(NewPtr)));
-}
-UnaryOperator *Context::createUnaryOperator(llvm::UnaryOperator *I) {
-  auto NewPtr = std::unique_ptr<UnaryOperator>(new UnaryOperator(I, *this));
-  return cast<UnaryOperator>(registerValue(std::move(NewPtr)));
-}
-BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
-  auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
-  return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
-}
-AtomicRMWInst *Context::createAtomicRMWInst(llvm::AtomicRMWInst *I) {
-  auto NewPtr = std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(I, *this));
-  return cast<AtomicRMWInst>(registerValue(std::move(NewPtr)));
-}
-AtomicCmpXchgInst *
-Context::createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I) {
-  auto NewPtr =
-      std::unique_ptr<AtomicCmpXchgInst>(new AtomicCmpXchgInst(I, *this));
-  return cast<AtomicCmpXchgInst>(registerValue(std::move(NewPtr)));
-}
-AllocaInst *Context::createAllocaInst(llvm::AllocaInst *I) {
-  auto NewPtr = std::unique_ptr<AllocaInst>(new AllocaInst(I, *this));
-  return cast<AllocaInst>(registerValue(std::move(NewPtr)));
-}
-CastInst *Context::createCastInst(llvm::CastInst *I) {
-  auto NewPtr = std::unique_ptr<CastInst>(new CastInst(I, *this));
-  return cast<CastInst>(registerValue(std::move(NewPtr)));
-}
-PHINode *Context::createPHINode(llvm::PHINode *I) {
-  auto NewPtr = std::unique_ptr<PHINode>(new PHINode(I, *this));
-  return cast<PHINode>(registerValue(std::move(NewPtr)));
-}
-ICmpInst *Context::createICmpInst(llvm::ICmpInst *I) {
-  auto NewPtr = std::unique_ptr<ICmpInst>(new ICmpInst(I, *this));
-  return cast<ICmpInst>(registerValue(std::move(NewPtr)));
-}
-FCmpInst *Context::createFCmpInst(llvm::FCmpInst *I) {
-  auto NewPtr = std::unique_ptr<FCmpInst>(new FCmpInst(I, *this));
-  return cast<FCmpInst>(registerValue(std::move(NewPtr)));
-}
-CmpInst *CmpInst::create(Predicate P, Value *S1, Value *S2,
-                         Instruction *InsertBefore, Context &Ctx,
-                         const Twine &Name) {
-  auto &Builder = Ctx.getLLVMIRBuilder();
-  Builder.SetInsertPoint(InsertBefore->getTopmostLLVMInstruction());
-  auto *LLVMI = Builder.CreateCmp(P, S1->Val, S2->Val, Name);
-  if (dyn_cast<llvm::ICmpInst>(LLVMI))
-    return Ctx.createICmpInst(cast<llvm::ICmpInst>(LLVMI));
-  return Ctx.createFCmpInst(cast<llvm::FCmpInst>(LLVMI));
-}
-CmpInst *CmpInst::createWithCopiedFlags(Predicate P, Value *S1, Value *S2,
-                                        const Instruction *F,
-                                        Instruction *InsertBefore, Context &Ctx,
-                                        const Twine &Name) {
-  CmpInst *Inst = create(P, S1, S2, InsertBefore, Ctx, Name);
-  cast<llvm::CmpInst>(Inst->Val)->copyIRFlags(F->Val);
-  return Inst;
-}
-
-Type *CmpInst::makeCmpResultType(Type *OpndType) {
-  if (auto *VT = dyn_cast<VectorType>(OpndType)) {
-    // TODO: Cleanup when we have more complete support for
-    // sandboxir::VectorType
-    return OpndType->getContext().getType(llvm::VectorType::get(
-        llvm::Type::getInt1Ty(OpndType->getContext().LLVMCtx),
-        cast<llvm::VectorType>(VT->LLVMTy)->getElementCount()));
-  }
-  return Type::getInt1Ty(OpndType->getContext());
-}
-
-void CmpInst::setPredicate(Predicate P) {
-  Ctx.getTracker()
-      .emplaceIfTracking<
-          GenericSetter<&CmpInst::getPredicate, &CmpInst::setPredicate>>(this);
-  cast<llvm::CmpInst>(Val)->setPredicate(P);
-}
-
-void CmpInst::swapOperands() {
-  if (ICmpInst *IC = dyn_cast<ICmpInst>(this))
-    IC->swapOperands();
-  else
-    cast<FCmpInst>(this)->swapOperands();
-}
-
-void ICmpInst::swapOperands() {
-  Ctx.getTracker().emplaceIfTracking<CmpSwapOperands>(this);
-  cast<llvm::ICmpInst>(Val)->swapOperands();
-}
-
-void FCmpInst::swapOperands() {
-  Ctx.getTracker().emplaceIfTracking<CmpSwapOperands>(this);
-  cast<llvm::FCmpInst>(Val)->swapOperands();
-}
-
-#ifndef NDEBUG
-void CmpInst::dumpOS(raw_ostream &OS) const {
-  dumpCommonPrefix(OS);
-  dumpCommonSuffix(OS);
-}
-
-void CmpInst::dump() const {
-  dumpOS(dbgs());
-  dbgs() << "\n";
-}
-#endif // NDEBUG
-
-Value *Context::getValue(llvm::Value *V) const {
-  auto It = LLVMValueToValueMap.find(V);
-  if (It != LLVMValueToValueMap.end())
-    return It->second.get();
-  return nullptr;
-}
-
-Function *Context::createFunction(llvm::Function *F) {
-  assert(getValue(F) == nullptr && "Already exists!");
-  auto NewFPtr = std::unique_ptr<Function>(new Function(F, *this));
-  auto *SBF = cast<Function>(registerValue(std::move(NewFPtr)));
-  // Create arguments.
-  for (auto &Arg : F->args())
-    getOrCreateArgument(&Arg);
-  // Create BBs.
-  for (auto &BB : *F)
-    createBasicBlock(&BB);
-  return SBF;
-}
-
-Function *BasicBlock::getParent() const {
-  auto *BB = cast<llvm::BasicBlock>(Val);
-  auto *F = BB->getParent();
-  if (F == nullptr)
-    // Detached
-    return nullptr;
-  return cast_or_null<Function>(Ctx.getValue(F));
-}
-
-void BasicBlock::buildBasicBlockFromLLVMIR(llvm::BasicBlock *LLVMBB) {
-  for (llvm::Instruction &IRef : reverse(*LLVMBB)) {
-    llvm::Instruction *I = &IRef;
-    Ctx.getOrCreateValue(I);
-    for (auto [OpIdx, Op] : enumerate(I->operands())) {
-      // Skip instruction's label operands
-      if (isa<llvm::BasicBlock>(Op))
-        continue;
-      // Skip metadata
-      if (isa<llvm::MetadataAsValue>(Op))
-        continue;
-      // Skip asm
-      if (isa<llvm::InlineAsm>(Op))
-        continue;
-      Ctx.getOrCreateValue(Op);
-    }
-  }
-#if !defined(NDEBUG)
-  verify();
-#endif
-}
-
-BasicBlock::iterator BasicBlock::begin() const {
-  llvm::BasicBlock *BB = cast<llvm::BasicBlock>(Val);
-  llvm::BasicBlock::iterator It = BB->begin();
-  if (!BB->empty()) {
-    auto *V = Ctx.getValue(&*BB->begin());
-    assert(V != nullptr && "No SandboxIR for BB->begin()!");
-    auto *I = cast<Instruction>(V);
-    unsigned Num = I->getNumOfIRInstrs();
-    assert(Num >= 1u && "Bad getNumOfIRInstrs()");
-    It = std::next(It, Num - 1);
-  }
-  return iterator(BB, It, &Ctx);
-}
-
-Instruction *BasicBlock::getTerminator() const {
-  auto *TerminatorV =
-      Ctx.getValue(cast<llvm::BasicBlock>(Val)->getTerminator());
-  return cast_or_null<Instruction>(TerminatorV);
-}
-
-Instruction &BasicBlock::front() const {
-  auto *BB = cast<llvm::BasicBlock>(Val);
-  assert(!BB->empty() && "Empty block!");
-  auto *SBI = cast<Instruction>(getContext().getValue(&*BB->begin()));
-  assert(SBI != nullptr && "Expected Instr!");
-  return *SBI;
-}
-
-Instruction &BasicBlock::back() const {
-  auto *BB = cast<llvm::BasicBlock>(Val);
-  assert(!BB->empty() && "Empty block!");
-  auto *SBI = cast<Instruction>(getContext().getValue(&*BB->rbegin()));
-  assert(SBI != nullptr && "Expected Instr!");
-  return *SBI;
-}
-
-#ifndef NDEBUG
-void BasicBlock::dumpOS(raw_ostream &OS) const {
-  llvm::BasicBlock *BB = cast<llvm::BasicBlock>(Val);
-  const auto &Name = BB->getName();
-  OS << Name;
-  if (!Name.empty())
-    OS << ":\n";
-  // If there are Instructions in the BB that are not mapped to SandboxIR, then
-  // use a crash-proof dump.
-  if (any_of(*BB, [this](llvm::Instruction &I) {
-        return Ctx.getValue(&I) == nullptr;
-      })) {
-    OS << "<Crash-proof mode!>\n";
-    DenseSet<Instruction *> Visited;
-    for (llvm::Instruction &IRef : *BB) {
-      Value *SBV = Ctx.getValue(&IRef);
-      if (SBV == nullptr)
-        OS << IRef << " *** No SandboxIR ***\n";
-      else {
-        auto *SBI = dyn_cast<Instruction>(SBV);
-        if (SBI == nullptr) {
-          OS << IRef << " *** Not a SBInstruction!!! ***\n";
-        } else {
-          if (Visited.insert(SBI).second)
-            OS << *SBI << "\n";
-        }
-      }
-    }
-  } else {
-    for (auto &SBI : *this) {
-      SBI.dumpOS(OS);
-      OS << "\n";
-    }
-  }
-}
-
-void BasicBlock::verify() const {
-  assert(isa<llvm::BasicBlock>(Val) && "Expected BasicBlock!");
-  for (const auto &I : *this) {
-    I.verify();
-  }
-}
-
-#endif // NDEBUG
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index b1f472d7928f4..abcad39330094 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -10,7 +10,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Instruction.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include <sstream>
 
 using namespace llvm::sandboxir;
diff --git a/llvm/lib/SandboxIR/Type.cpp b/llvm/lib/SandboxIR/Type.cpp
index 87dcb726dde35..7bb788ecf25a6 100644
--- a/llvm/lib/SandboxIR/Type.cpp
+++ b/llvm/lib/SandboxIR/Type.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Type.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Context.h"
 
 using namespace llvm::sandboxir;
 
diff --git a/llvm/lib/SandboxIR/User.cpp b/llvm/lib/SandboxIR/User.cpp
new file mode 100644
index 0000000000000..8afa52e32b762
--- /dev/null
+++ b/llvm/lib/SandboxIR/User.cpp
@@ -0,0 +1,74 @@
+//===- User.cpp - The User class of Sandbox IR ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/User.h"
+#include "llvm/SandboxIR/Context.h"
+
+namespace llvm::sandboxir {
+
+Use User::getOperandUseDefault(unsigned OpIdx, bool Verify) const {
+  assert((!Verify || OpIdx < getNumOperands()) && "Out of bounds!");
+  assert(isa<llvm::User>(Val) && "Non-users have no operands!");
+  llvm::Use *LLVMUse;
+  if (OpIdx != getNumOperands())
+    LLVMUse = &cast<llvm::User>(Val)->getOperandUse(OpIdx);
+  else
+    LLVMUse = cast<llvm::User>(Val)->op_end();
+  return Use(LLVMUse, const_cast<User *>(this), Ctx);
+}
+
+#ifndef NDEBUG
+void User::verifyUserOfLLVMUse(const llvm::Use &Use) const {
+  assert(Ctx.getValue(Use.getUser()) == this &&
+         "Use not found in this SBUser's operands!");
+}
+#endif
+
+bool User::classof(const Value *From) {
+  switch (From->getSubclassID()) {
+#define DEF_VALUE(ID, CLASS)
+#define DEF_USER(ID, CLASS)                                                    \
+  case ClassID::ID:                                                            \
+    return true;
+#define DEF_INSTR(ID, OPC, CLASS)                                              \
+  case ClassID::ID:                                                            \
+    return true;
+#include "llvm/SandboxIR/SandboxIRValues.def"
+  default:
+    return false;
+  }
+}
+
+void User::setOperand(unsigned OperandIdx, Value *Operand) {
+  assert(isa<llvm::User>(Val) && "No operands!");
+  Ctx.getTracker().emplaceIfTracking<UseSet>(getOperandUse(OperandIdx));
+  // We are delegating to llvm::User::setOperand().
+  cast<llvm::User>(Val)->setOperand(OperandIdx, Operand->Val);
+}
+
+bool User::replaceUsesOfWith(Value *FromV, Value *ToV) {
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking()) {
+    for (auto OpIdx : seq<unsigned>(0, getNumOperands())) {
+      auto Use = getOperandUse(OpIdx);
+      if (Use.get() == FromV)
+        Tracker.emplaceIfTracking<UseSet>(Use);
+    }
+  }
+  // We are delegating RUOW to LLVM IR's RUOW.
+  return cast<llvm::User>(Val)->replaceUsesOfWith(FromV->Val, ToV->Val);
+}
+
+#ifndef NDEBUG
+void User::dumpCommonHeader(raw_ostream &OS) const {
+  Value::dumpCommonHeader(OS);
+  // TODO: This is incomplete
+}
+#endif // NDEBUG
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/SandboxIR/Value.cpp b/llvm/lib/SandboxIR/Value.cpp
new file mode 100644
index 0000000000000..b9d91c7e11f74
--- /dev/null
+++ b/llvm/lib/SandboxIR/Value.cpp
@@ -0,0 +1,122 @@
+//===- Value.cpp - The Value class of Sandbox IR --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/SandboxIR/Value.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/User.h"
+#include <sstream>
+
+namespace llvm::sandboxir {
+
+Value::Value(ClassID SubclassID, llvm::Value *Val, Context &Ctx)
+    : SubclassID(SubclassID), Val(Val), Ctx(Ctx) {
+#ifndef NDEBUG
+  UID = Ctx.getNumValues();
+#endif
+}
+
+Value::use_iterator Value::use_begin() {
+  llvm::Use *LLVMUse = nullptr;
+  if (Val->use_begin() != Val->use_end())
+    LLVMUse = &*Val->use_begin();
+  User *User = LLVMUse != nullptr ? cast_or_null<sandboxir::User>(Ctx.getValue(
+                                        Val->use_begin()->getUser()))
+                                  : nullptr;
+  return use_iterator(Use(LLVMUse, User, Ctx));
+}
+
+Value::user_iterator Value::user_begin() {
+  auto UseBegin = Val->use_begin();
+  auto UseEnd = Val->use_end();
+  bool AtEnd = UseBegin == UseEnd;
+  llvm::Use *LLVMUse = AtEnd ? nullptr : &*UseBegin;
+  User *User =
+      AtEnd ? nullptr
+            : cast_or_null<sandboxir::User>(Ctx.getValue(&*LLVMUse->getUser()));
+  return user_iterator(Use(LLVMUse, User, Ctx), UseToUser());
+}
+
+unsigned Value::getNumUses() const { return range_size(Val->users()); }
+
+Type *Value::getType() const { return Ctx.getType(Val->getType()); }
+
+void Value::replaceUsesWithIf(
+    Value *OtherV, llvm::function_ref<bool(const Use &)> ShouldReplace) {
+  assert(getType() == OtherV->getType() && "Can't replace with different type");
+  llvm::Value *OtherVal = OtherV->Val;
+  // We are delegating RUWIf to LLVM IR's RUWIf.
+  Val->replaceUsesWithIf(
+      OtherVal, [&ShouldReplace, this](llvm::Use &LLVMUse) -> bool {
+        User *DstU = cast_or_null<User>(Ctx.getValue(LLVMUse.getUser()));
+        if (DstU == nullptr)
+          return false;
+        Use UseToReplace(&LLVMUse, DstU, Ctx);
+        if (!ShouldReplace(UseToReplace))
+          return false;
+        Ctx.getTracker().emplaceIfTracking<UseSet>(UseToReplace);
+        return true;
+      });
+}
+
+void Value::replaceAllUsesWith(Value *Other) {
+  assert(getType() == Other->getType() &&
+         "Replacing with Value of different type!");
+  auto &Tracker = Ctx.getTracker();
+  if (Tracker.isTracking()) {
+    for (auto Use : uses())
+      Tracker.track(std::make_unique<UseSet>(Use));
+  }
+  // We are delegating RAUW to LLVM IR's RAUW.
+  Val->replaceAllUsesWith(Other->Val);
+}
+
+#ifndef NDEBUG
+std::string Value::getUid() const {
+  std::stringstream SS;
+  SS << "SB" << UID << ".";
+  return SS.str();
+}
+
+void Value::dumpCommonHeader(raw_ostream &OS) const {
+  OS << getUid() << " " << getSubclassIDStr(SubclassID) << " ";
+}
+
+void Value::dumpCommonFooter(raw_ostream &OS) const {
+  OS.indent(2) << "Val: ";
+  if (Val)
+    OS << *Val;
+  else
+    OS << "NULL";
+  OS << "\n";
+}
+
+void Value::dumpCommonPrefix(raw_ostream &OS) const {
+  if (Val)
+    OS << *Val;
+  else
+    OS << "NULL ";
+}
+
+void Value::dumpCommonSuffix(raw_ostream &OS) const {
+  OS << " ; " << getUid() << " (" << getSubclassIDStr(SubclassID) << ")";
+}
+
+void Value::printAsOperandCommon(raw_ostream &OS) const {
+  if (Val)
+    Val->printAsOperand(OS);
+  else
+    OS << "NULL ";
+}
+
+void Value::dump() const {
+  dumpOS(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
+} // namespace llvm::sandboxir
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 8e31e0ced2d73..6863c5c0af5dc 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -1075,9 +1075,13 @@ KnownBits KnownBits::srem(const KnownBits &LHS, const KnownBits &RHS) {
 
   // The sign bit is the LHS's sign bit, except when the result of the
   // remainder is zero. The magnitude of the result should be less than or
-  // equal to the magnitude of the LHS. Therefore any leading zeros that exist
-  // in the left hand side must also exist in the result.
-  Known.Zero.setHighBits(LHS.countMinLeadingZeros());
+  // equal to the magnitude of either operand.
+  if (LHS.isNegative() && Known.isNonZero())
+    Known.One.setHighBits(
+        std::max(LHS.countMinLeadingOnes(), RHS.countMinSignBits()));
+  else if (LHS.isNonNegative())
+    Known.Zero.setHighBits(
+        std::max(LHS.countMinLeadingZeros(), RHS.countMinSignBits()));
   return Known;
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index c6e88131d5a34..a82a081e4abfe 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -164,6 +164,8 @@ class AArch64AsmPrinter : public AsmPrinter {
   /// pseudo instructions.
   bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
 
+  void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
+
   void emitInstruction(const MachineInstr *MI) override;
 
   void emitFunctionHeaderComment() override;
@@ -229,6 +231,10 @@ class AArch64AsmPrinter : public AsmPrinter {
   /// Emit the LOHs contained in AArch64FI.
   void emitLOHs();
 
+  void emitMovXReg(Register Dest, Register Src);
+  void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
+  void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
+
   /// Emit instruction to set float register to zero.
   void emitFMov0(const MachineInstr &MI);
 
@@ -409,16 +415,6 @@ void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
   auto &O = *OutStreamer;
   MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
   O.emitLabel(CurSled);
-  MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
-                        .addReg(AArch64::X0)
-                        .addReg(AArch64::XZR)
-                        .addReg(MI.getOperand(0).getReg())
-                        .addImm(0);
-  MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
-                        .addReg(AArch64::X1)
-                        .addReg(AArch64::XZR)
-                        .addReg(MI.getOperand(1).getReg())
-                        .addImm(0);
   bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
   auto *Sym = MCSymbolRefExpr::create(
       OutContext.getOrCreateSymbol(
@@ -438,13 +434,9 @@ void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
                           .addReg(AArch64::X2)
                           .addReg(AArch64::SP)
                           .addImm(2));
-    EmitToStreamer(O, MovX0Op0);
-    EmitToStreamer(O, MovX1Op1);
-    EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs)
-                          .addReg(AArch64::X2)
-                          .addReg(AArch64::XZR)
-                          .addReg(MI.getOperand(2).getReg())
-                          .addImm(0));
+    emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
+    emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
+    emitMovXReg(AArch64::X2, MI.getOperand(2).getReg());
     EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
     EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
                           .addReg(AArch64::X2)
@@ -468,8 +460,8 @@ void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
                           .addReg(AArch64::X1)
                           .addReg(AArch64::SP)
                           .addImm(-2));
-    EmitToStreamer(O, MovX0Op0);
-    EmitToStreamer(O, MovX1Op1);
+    emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
+    emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
     EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
     O.AddComment("End XRay custom event");
     EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
@@ -497,11 +489,7 @@ void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
     // Checking XZR makes no sense. Instead of emitting a load, zero
     // ScratchRegs[0] and use it for the ESR AddrIndex below.
     AddrReg = getXRegFromWReg(ScratchRegs[0]);
-    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
-                                     .addReg(AddrReg)
-                                     .addReg(AArch64::XZR)
-                                     .addReg(AArch64::XZR)
-                                     .addImm(0));
+    emitMovXReg(AddrReg, AArch64::XZR);
   } else {
     // If one of the scratch registers is used for the call target (e.g.
     // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
@@ -534,16 +522,8 @@ void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
 
   // Load the expected type hash.
   const int64_t Type = MI.getOperand(1).getImm();
-  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
-                                   .addReg(ScratchRegs[1])
-                                   .addReg(ScratchRegs[1])
-                                   .addImm(Type & 0xFFFF)
-                                   .addImm(0));
-  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
-                                   .addReg(ScratchRegs[1])
-                                   .addReg(ScratchRegs[1])
-                                   .addImm((Type >> 16) & 0xFFFF)
-                                   .addImm(16));
+  emitMOVK(ScratchRegs[1], Type & 0xFFFF, 0);
+  emitMOVK(ScratchRegs[1], (Type >> 16) & 0xFFFF, 16);
 
   // Compare the hashes and trap if there's a mismatch.
   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
@@ -627,6 +607,7 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
   std::unique_ptr<MCSubtargetInfo> STI(
       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
   assert(STI && "Unable to create subtarget info");
+  this->STI = static_cast<const AArch64Subtarget *>(&*STI);
 
   MCSymbol *HwasanTagMismatchV1Sym =
       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
@@ -679,11 +660,7 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
       // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
       // left-shift option in the MOV instruction. Combined with the 16-bit
       // immediate, this is enough to represent any offset up to 2**48.
-      OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
-                                       .addReg(AArch64::X17)
-                                       .addImm(FixedShadowOffset >> 32)
-                                       .addImm(32),
-                                   *STI);
+      emitMOVZ(AArch64::X17, FixedShadowOffset >> 32, 32);
       OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
                                        .addReg(AArch64::W16)
                                        .addReg(AArch64::X17)
@@ -823,18 +800,8 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
                                  *STI);
 
     if (Reg != AArch64::X0)
-      OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
-                                       .addReg(AArch64::X0)
-                                       .addReg(AArch64::XZR)
-                                       .addReg(Reg)
-                                       .addImm(0),
-                                   *STI);
-    OutStreamer->emitInstruction(
-        MCInstBuilder(AArch64::MOVZXi)
-            .addReg(AArch64::X1)
-            .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
-            .addImm(0),
-        *STI);
+      emitMovXReg(AArch64::X0, Reg);
+    emitMOVZ(AArch64::X1, AccessInfo & HWASanAccessInfo::RuntimeMask, 0);
 
     if (CompileKernel) {
       // The Linux kernel's dynamic loader doesn't support GOT relative
@@ -865,6 +832,7 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
           MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
     }
   }
+  this->STI = nullptr;
 }
 
 static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
@@ -1438,11 +1406,7 @@ void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
                                      .addImm(0));
     ++InstsEmitted;
   } else {
-    EmitToStreamer(*OutStreamer,
-                   MCInstBuilder(AArch64::MOVZXi)
-                       .addReg(AArch64::X17)
-                       .addImm(static_cast<uint16_t>(MaxTableEntry))
-                       .addImm(0));
+    emitMOVZ(AArch64::X17, static_cast<uint16_t>(MaxTableEntry), 0);
     ++InstsEmitted;
     // It's sad that we have to manually materialize instructions, but we can't
     // trivially reuse the main pseudo expansion logic.
@@ -1450,12 +1414,8 @@ void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
     for (int Offset = 16; Offset < 64; Offset += 16) {
       if ((MaxTableEntry >> Offset) == 0)
         break;
-      EmitToStreamer(*OutStreamer,
-                     MCInstBuilder(AArch64::MOVKXi)
-                         .addReg(AArch64::X17)
-                         .addReg(AArch64::X17)
-                         .addImm(static_cast<uint16_t>(MaxTableEntry >> Offset))
-                         .addImm(Offset));
+      emitMOVK(AArch64::X17, static_cast<uint16_t>(MaxTableEntry >> Offset),
+               Offset);
       ++InstsEmitted;
     }
     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
@@ -1615,20 +1575,9 @@ void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
     Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
     EncodedBytes = 16;
     // Materialize the jump address:
-    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
-                                    .addReg(ScratchReg)
-                                    .addImm((CallTarget >> 32) & 0xFFFF)
-                                    .addImm(32));
-    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
-                                    .addReg(ScratchReg)
-                                    .addReg(ScratchReg)
-                                    .addImm((CallTarget >> 16) & 0xFFFF)
-                                    .addImm(16));
-    EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
-                                    .addReg(ScratchReg)
-                                    .addReg(ScratchReg)
-                                    .addImm(CallTarget & 0xFFFF)
-                                    .addImm(0));
+    emitMOVZ(ScratchReg, (CallTarget >> 32) & 0xFFFF, 32);
+    emitMOVK(ScratchReg, (CallTarget >> 16) & 0xFFFF, 16);
+    emitMOVK(ScratchReg, CallTarget & 0xFFFF, 0);
     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
   }
   // Emit padding.
@@ -1717,6 +1666,33 @@ void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
   OutStreamer->emitInstruction(MI, getSubtargetInfo());
 }
 
+void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
+  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+                                   .addReg(Dest)
+                                   .addReg(AArch64::XZR)
+                                   .addReg(Src)
+                                   .addImm(0));
+}
+
+void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
+  bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
+                     .addReg(Dest)
+                     .addImm(Imm)
+                     .addImm(Shift));
+}
+
+void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
+  bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
+                     .addReg(Dest)
+                     .addReg(Dest)
+                     .addImm(Imm)
+                     .addImm(Shift));
+}
+
 void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
   Register DestReg = MI.getOperand(0).getReg();
   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
@@ -1774,26 +1750,15 @@ unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
 
   // If there's only a constant discriminator, MOV it into x17.
   if (AddrDisc == AArch64::XZR) {
-    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
-                                     .addReg(AArch64::X17)
-                                     .addImm(Disc)
-                                     .addImm(/*shift=*/0));
+    emitMOVZ(AArch64::X17, Disc, 0);
     ++InstsEmitted;
     return AArch64::X17;
   }
 
   // If there are both, emit a blend into x17.
-  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
-                                   .addReg(AArch64::X17)
-                                   .addReg(AArch64::XZR)
-                                   .addReg(AddrDisc)
-                                   .addImm(0));
+  emitMovXReg(AArch64::X17, AddrDisc);
   ++InstsEmitted;
-  EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
-                                   .addReg(AArch64::X17)
-                                   .addReg(AArch64::X17)
-                                   .addImm(Disc)
-                                   .addImm(/*shift=*/48));
+  emitMOVK(AArch64::X17, Disc, 48);
   ++InstsEmitted;
   return AArch64::X17;
 }
@@ -1914,11 +1879,7 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
 
     // XPAC has tied src/dst: use x17 as a temporary copy.
     //  mov x17, x16
-    EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
-                                     .addReg(AArch64::X17)
-                                     .addReg(AArch64::XZR)
-                                     .addReg(AArch64::X16)
-                                     .addImm(0));
+    emitMovXReg(AArch64::X17, AArch64::X16);
     ++InstsEmitted;
 
     //  xpaci x17
@@ -1955,11 +1916,7 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
       // FIXME: can we simply return the AUT result, already in x16? without..
       //        ..traps this is usable as an oracle anyway, based on high bits
       //  mov x17, x16
-      EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
-                                       .addReg(AArch64::X16)
-                                       .addReg(AArch64::XZR)
-                                       .addReg(AArch64::X17)
-                                       .addImm(0));
+      emitMovXReg(AArch64::X16, AArch64::X17);
       ++InstsEmitted;
 
       if (IsAUTPAC) {
@@ -2273,13 +2230,9 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
             return true;
         return false;
       };
-      for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) {
-        EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
-                             .addReg(AArch64::X17)
-                             .addReg(AArch64::X17)
-                             .addImm((UOffset >> BitPos) & 0xffff)
-                             .addImm(/*shift=*/BitPos));
-      }
+      for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
+        emitMOVK(AArch64::X17, (UOffset >> BitPos) & 0xffff, BitPos);
+
       EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs)
                            .addReg(AArch64::X16)
                            .addReg(AArch64::X16)
@@ -2291,21 +2244,10 @@ void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
   unsigned DiscReg = AddrDisc;
   if (Disc != 0) {
     if (AddrDisc != AArch64::XZR) {
-      EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
-                           .addReg(AArch64::X17)
-                           .addReg(AArch64::XZR)
-                           .addReg(AddrDisc)
-                           .addImm(0));
-      EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
-                           .addReg(AArch64::X17)
-                           .addReg(AArch64::X17)
-                           .addImm(Disc)
-                           .addImm(/*shift=*/48));
+      emitMovXReg(AArch64::X17, AddrDisc);
+      emitMOVK(AArch64::X17, Disc, 48);
     } else {
-      EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi)
-                           .addReg(AArch64::X17)
-                           .addImm(Disc)
-                           .addImm(/*shift=*/0));
+      emitMOVZ(AArch64::X17, Disc, 0);
     }
     DiscReg = AArch64::X17;
   }
@@ -2337,6 +2279,10 @@ AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
 // instructions) auto-generated.
 #include "AArch64GenMCPseudoLowering.inc"
 
+void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
+  S.emitInstruction(Inst, *STI);
+}
+
 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
   AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
 
@@ -2511,21 +2457,10 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
     if (Disc) {
       if (AddrDisc != AArch64::NoRegister) {
         if (ScratchReg != AddrDisc)
-          EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
-                                           .addReg(ScratchReg)
-                                           .addReg(AArch64::XZR)
-                                           .addReg(AddrDisc)
-                                           .addImm(0));
-        EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
-                                         .addReg(ScratchReg)
-                                         .addReg(ScratchReg)
-                                         .addImm(Disc)
-                                         .addImm(/*shift=*/48));
+          emitMovXReg(ScratchReg, AddrDisc);
+        emitMOVK(ScratchReg, Disc, 48);
       } else {
-        EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
-                                         .addReg(ScratchReg)
-                                         .addImm(Disc)
-                                         .addImm(/*shift=*/0));
+        emitMOVZ(ScratchReg, Disc, 0);
       }
       DiscReg = ScratchReg;
     }
diff --git a/llvm/lib/Target/AArch64/AArch64FMV.td b/llvm/lib/Target/AArch64/AArch64FMV.td
index 16468068d7a8f..8266507379f3b 100644
--- a/llvm/lib/Target/AArch64/AArch64FMV.td
+++ b/llvm/lib/Target/AArch64/AArch64FMV.td
@@ -73,7 +73,6 @@ def : FMVExtension<"rdm", "FEAT_RDM", "+rdm,+fp-armv8,+neon", 108>;
 def : FMVExtension<"rng", "FEAT_RNG", "+rand", 10>;
 def : FMVExtension<"rpres", "FEAT_RPRES", "", 300>;
 def : FMVExtension<"sb", "FEAT_SB", "+sb", 470>;
-def : FMVExtension<"sha1", "FEAT_SHA1", "+fp-armv8,+neon", 120>;
 def : FMVExtension<"sha2", "FEAT_SHA2", "+sha2,+fp-armv8,+neon", 130>;
 def : FMVExtension<"sha3", "FEAT_SHA3", "+sha3,+sha2,+fp-armv8,+neon", 140>;
 def : FMVExtension<"simd", "FEAT_SIMD", "+fp-armv8,+neon", 100>;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 46b462de5071c..1d1d9b5512cfc 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -6243,6 +6243,26 @@ multiclass SIMDThreeSameVectorDOT4<string asm> {
                                           V128, v4f32, v16i8, null_frag>;
 }
 
+let mayRaiseFPException = 1, Uses = [FPCR] in
+multiclass SIMDThreeVectorFscale<bit U, bit S, bits<3> opc,
+                                 string asm, SDPatternOperator OpNode> {
+  def v4f16 : BaseSIMDThreeSameVector<0, U, {S,0b10}, {0b00,opc}, V64,
+                                      asm, ".4h",
+        [(set (v4f16 V64:$Rd), (OpNode (v4f16 V64:$Rn), (v4i16 V64:$Rm)))]>;
+  def v8f16 : BaseSIMDThreeSameVector<1, U, {S,0b10}, {0b00,opc}, V128,
+                                      asm, ".8h",
+        [(set (v8f16 V128:$Rd), (OpNode (v8f16 V128:$Rn), (v8i16 V128:$Rm)))]>;
+  def v2f32 : BaseSIMDThreeSameVector<0, U, {S,0b01}, {0b11,opc}, V64,
+                                      asm, ".2s",
+        [(set (v2f32 V64:$Rd), (OpNode (v2f32 V64:$Rn), (v2i32 V64:$Rm)))]>;
+  def v4f32 : BaseSIMDThreeSameVector<1, U, {S,0b01}, {0b11,opc}, V128,
+                                      asm, ".4s",
+        [(set (v4f32 V128:$Rd), (OpNode (v4f32 V128:$Rn), (v4i32 V128:$Rm)))]>;
+  def v2f64 : BaseSIMDThreeSameVector<1, U, {S,0b11}, {0b11,opc}, V128,
+                                      asm, ".2d",
+        [(set (v2f64 V128:$Rd), (OpNode (v2f64 V128:$Rn), (v2i64 V128:$Rm)))]>;
+}
+
 //----------------------------------------------------------------------------
 // AdvSIMD two register vector instructions.
 //----------------------------------------------------------------------------
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index c70e835d1619f..943c48c0f230a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -10136,7 +10136,7 @@ let Uses = [FPMR, FPCR], Predicates = [HasFP8] in {
   defm BF2CVTL : SIMDMixedTwoVectorFP8<0b11, "bf2cvtl">;
   defm FCVTN_F16_F8 : SIMDThreeSameSizeVectorCvt<"fcvtn">;
   defm FCVTN_F32_F8 : SIMDThreeVectorCvt<"fcvtn">;
-  defm FSCALE : SIMDThreeSameVectorFP<0b1, 0b1, 0b111, "fscale", null_frag>;
+  defm FSCALE : SIMDThreeVectorFscale<0b1, 0b1, 0b111, "fscale", int_aarch64_neon_fp8_fscale>;
 } // End let Predicates = [HasFP8]
 
 // fminimum(abs(a), abs(b)) -> famin(a, b)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 6cb181011f8f6..51aeee023f2e3 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -268,11 +268,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       })
       .scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
       .lowerIf(scalarOrEltWiderThan(0, 64))
-      .minScalarOrElt(0, MinFPScalar)
       .clampNumElements(0, v4s16, v8s16)
       .clampNumElements(0, v2s32, v4s32)
       .clampNumElements(0, v2s64, v2s64)
-      .moreElementsToNextPow2(0);
+      .moreElementsToNextPow2(0)
+      .lowerFor({s16, v4s16, v8s16});
 
   getActionDefinitionsBuilder(G_FREM)
       .libcallFor({s32, s64})
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 97c5f96388abe..c7f44ec018f5a 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -447,10 +447,10 @@ class AArch64MCInstrAnalysis : public MCInstrAnalysis {
     const MCRegisterClass &FPR128RC =
         MRI.getRegClass(AArch64::FPR128RegClassID);
 
-    auto ClearsSuperReg = [=](unsigned RegID) {
+    auto ClearsSuperReg = [=](MCRegister Reg) {
       // An update to the lower 32 bits of a 64 bit integer register is
       // architecturally defined to zero extend the upper 32 bits on a write.
-      if (GPR32RC.contains(RegID))
+      if (GPR32RC.contains(Reg))
         return true;
       // SIMD&FP instructions operating on scalar data only acccess the lower
       // bits of a register, the upper bits are zero extended on a write. For
@@ -458,9 +458,9 @@ class AArch64MCInstrAnalysis : public MCInstrAnalysis {
       // register are zero extended on a write.
       // When VL is higher than 128 bits, any write to a SIMD&FP register sets
       // bits higher than 128 to zero.
-      return FPR8RC.contains(RegID) || FPR16RC.contains(RegID) ||
-             FPR32RC.contains(RegID) || FPR64RC.contains(RegID) ||
-             FPR128RC.contains(RegID);
+      return FPR8RC.contains(Reg) || FPR16RC.contains(Reg) ||
+             FPR32RC.contains(Reg) || FPR64RC.contains(Reg) ||
+             FPR128RC.contains(Reg);
     };
 
     Mask.clearAllBits();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index fad51ce8285e0..94fdf4effa10a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -900,6 +900,10 @@ bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode *N) const {
     unsigned IntrID = N->getConstantOperandVal(0);
     return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
   }
+  case ISD::INTRINSIC_W_CHAIN: {
+    unsigned IntrID = N->getConstantOperandVal(1);
+    return AMDGPU::isIntrinsicAlwaysUniform(IntrID);
+  }
   case ISD::LOAD:
     if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() ==
         AMDGPUAS::CONSTANT_ADDRESS_32BIT)
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 555b8cb5c6e53..1308e18f88b9d 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1838,10 +1838,8 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   ParseStatus parseSOPPBrTarget(OperandVector &Operands);
   ParseStatus parseBoolReg(OperandVector &Operands);
 
-  bool parseSwizzleOperand(int64_t &Op,
-                           const unsigned MinVal,
-                           const unsigned MaxVal,
-                           const StringRef ErrMsg,
+  bool parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
+                           const unsigned MaxVal, const Twine &ErrMsg,
                            SMLoc &Loc);
   bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
                             const unsigned MinVal,
@@ -1855,6 +1853,8 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
   bool parseSwizzleBroadcast(int64_t &Imm);
   bool parseSwizzleSwap(int64_t &Imm);
   bool parseSwizzleReverse(int64_t &Imm);
+  bool parseSwizzleFFT(int64_t &Imm);
+  bool parseSwizzleRotate(int64_t &Imm);
 
   ParseStatus parseGPRIdxMode(OperandVector &Operands);
   int64_t parseGPRIdxMacro();
@@ -7996,12 +7996,9 @@ encodeBitmaskPerm(const unsigned AndMask,
          (XorMask << BITMASK_XOR_SHIFT);
 }
 
-bool
-AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op,
-                                     const unsigned MinVal,
-                                     const unsigned MaxVal,
-                                     const StringRef ErrMsg,
-                                     SMLoc &Loc) {
+bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
+                                          const unsigned MaxVal,
+                                          const Twine &ErrMsg, SMLoc &Loc) {
   if (!skipToken(AsmToken::Comma, "expected a comma")) {
     return false;
   }
@@ -8166,6 +8163,54 @@ AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
   return true;
 }
 
+bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &Imm) {
+  using namespace llvm::AMDGPU::Swizzle;
+
+  if (!AMDGPU::isGFX9Plus(getSTI())) {
+    Error(getLoc(), "FFT mode swizzle not supported on this GPU");
+    return false;
+  }
+
+  int64_t Swizzle;
+  SMLoc Loc;
+  if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
+                           "FFT swizzle must be in the interval [0," +
+                               Twine(FFT_SWIZZLE_MAX) + Twine(']'),
+                           Loc))
+    return false;
+
+  Imm = FFT_MODE_ENC | Swizzle;
+  return true;
+}
+
+bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &Imm) {
+  using namespace llvm::AMDGPU::Swizzle;
+
+  if (!AMDGPU::isGFX9Plus(getSTI())) {
+    Error(getLoc(), "Rotate mode swizzle not supported on this GPU");
+    return false;
+  }
+
+  SMLoc Loc;
+  int64_t Direction;
+
+  if (!parseSwizzleOperand(Direction, 0, 1,
+                           "direction must be 0 (left) or 1 (right)", Loc))
+    return false;
+
+  int64_t RotateSize;
+  if (!parseSwizzleOperand(
+          RotateSize, 0, ROTATE_MAX_SIZE,
+          "number of threads to rotate must be in the interval [0," +
+              Twine(ROTATE_MAX_SIZE) + Twine(']'),
+          Loc))
+    return false;
+
+  Imm = ROTATE_MODE_ENC | (Direction << ROTATE_DIR_SHIFT) |
+        (RotateSize << ROTATE_SIZE_SHIFT);
+  return true;
+}
+
 bool
 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
 
@@ -8200,6 +8245,10 @@ AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
       Ok = parseSwizzleSwap(Imm);
     } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
       Ok = parseSwizzleReverse(Imm);
+    } else if (trySkipId(IdSymbolic[ID_FFT])) {
+      Ok = parseSwizzleFFT(Imm);
+    } else if (trySkipId(IdSymbolic[ID_ROTATE])) {
+      Ok = parseSwizzleRotate(Imm);
     } else {
       Error(ModeLoc, "expected a swizzle mode");
     }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index 94bf5e4b95270..91e8f52df5b41 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -1503,8 +1503,21 @@ void AMDGPUInstPrinter::printSwizzle(const MCInst *MI, unsigned OpNo,
 
   O << " offset:";
 
-  if ((Imm & QUAD_PERM_ENC_MASK) == QUAD_PERM_ENC) {
+  // Rotate and FFT modes
+  if (Imm >= ROTATE_MODE_LO && AMDGPU::isGFX9Plus(STI)) {
+    if (Imm >= FFT_MODE_LO) {
+      O << "swizzle(" << IdSymbolic[ID_FFT] << ',' << (Imm & FFT_SWIZZLE_MASK)
+        << ')';
+    } else if (Imm >= ROTATE_MODE_LO) {
+      O << "swizzle(" << IdSymbolic[ID_ROTATE] << ','
+        << ((Imm >> ROTATE_DIR_SHIFT) & ROTATE_DIR_MASK) << ','
+        << ((Imm >> ROTATE_SIZE_SHIFT) & ROTATE_SIZE_MASK) << ')';
+    }
+    return;
+  }
 
+  // Basic mode
+  if ((Imm & QUAD_PERM_ENC_MASK) == QUAD_PERM_ENC) {
     O << "swizzle(" << IdSymbolic[ID_QUAD_PERM];
     for (unsigned I = 0; I < LANE_NUM; ++I) {
       O << ",";
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index fb3d83ca30d19..07c80bd2575f0 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -841,9 +841,12 @@ enum Id : unsigned { // id of symbolic names
   ID_BITMASK_PERM,
   ID_SWAP,
   ID_REVERSE,
-  ID_BROADCAST
+  ID_BROADCAST,
+  ID_FFT,
+  ID_ROTATE
 };
 
+// clang-format off
 enum EncBits : unsigned {
 
   // swizzle mode encodings
@@ -854,6 +857,14 @@ enum EncBits : unsigned {
   BITMASK_PERM_ENC      = 0x0000,
   BITMASK_PERM_ENC_MASK = 0x8000,
 
+  FFT_MODE_ENC          = 0xE000,
+
+  ROTATE_MODE_ENC       = 0xC000,
+  FFT_ROTATE_MODE_MASK  = 0xF000,
+
+  ROTATE_MODE_LO        = 0xC000,
+  FFT_MODE_LO           = 0xE000,
+
   // QUAD_PERM encodings
 
   LANE_MASK             = 0x3,
@@ -869,8 +880,21 @@ enum EncBits : unsigned {
 
   BITMASK_AND_SHIFT     = 0,
   BITMASK_OR_SHIFT      = 5,
-  BITMASK_XOR_SHIFT     = 10
+  BITMASK_XOR_SHIFT     = 10,
+
+  // FFT encodings
+
+  FFT_SWIZZLE_MASK      = 0x1F,
+  FFT_SWIZZLE_MAX       = 0x1F,
+
+  // ROTATE encodings
+  ROTATE_MAX_SIZE       = 0x1F,
+  ROTATE_DIR_SHIFT      = 10, // bit position of rotate direction
+  ROTATE_DIR_MASK       = 0x1,
+  ROTATE_SIZE_SHIFT     = 5, // bit position of rotate size
+  ROTATE_SIZE_MASK      = ROTATE_MAX_SIZE,
 };
+// clang-format on
 
 } // namespace Swizzle
 
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 50a6f028f66de..07505110476b5 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1342,20 +1342,10 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
   SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
 
   // Allocate spill slots for WWM reserved VGPRs.
-  // For chain functions, we only need to do this if we have calls to
-  // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
-  // chain functions do not return) and the function did not contain a call to
-  // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
-  // when entering the function).
-  bool IsChainWithoutRestores =
-      FuncInfo->isChainFunction() &&
-      (!MF.getFrameInfo().hasTailCall() || FuncInfo->hasInitWholeWave());
-  if (!FuncInfo->isEntryFunction() && !IsChainWithoutRestores) {
-    for (Register Reg : FuncInfo->getWWMReservedRegs()) {
-      const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
-      FuncInfo->allocateWWMSpill(MF, Reg, TRI->getSpillSize(*RC),
-                                 TRI->getSpillAlign(*RC));
-    }
+  for (Register Reg : FuncInfo->getWWMReservedRegs()) {
+    const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
+    FuncInfo->allocateWWMSpill(MF, Reg, TRI->getSpillSize(*RC),
+                               TRI->getSpillAlign(*RC));
   }
 
   const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs()
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 885ecab891b1f..eb2f20f89de08 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -8209,9 +8209,17 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     append_range(Ops, VAddrs);
   else
     Ops.push_back(VAddr);
-  Ops.push_back(Op.getOperand(ArgOffset + Intr->RsrcIndex));
-  if (BaseOpcode->Sampler)
-    Ops.push_back(Op.getOperand(ArgOffset + Intr->SampIndex));
+  SDValue Rsrc = Op.getOperand(ArgOffset + Intr->RsrcIndex);
+  EVT RsrcVT = Rsrc.getValueType();
+  if (RsrcVT != MVT::v4i32 && RsrcVT != MVT::v8i32)
+    return Op;
+  Ops.push_back(Rsrc);
+  if (BaseOpcode->Sampler) {
+    SDValue Samp = Op.getOperand(ArgOffset + Intr->SampIndex);
+    if (Samp.getValueType() != MVT::v4i32)
+      return Op;
+    Ops.push_back(Samp);
+  }
   Ops.push_back(DAG.getTargetConstant(DMask, DL, MVT::i32));
   if (IsGFX10Plus)
     Ops.push_back(DAG.getTargetConstant(DimInfo->Encoding, DL, MVT::i32));
diff --git a/llvm/lib/Target/AMDGPU/SIInstrFormats.td b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
index 9b506eb0a711a..dd1ab2c628715 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
@@ -84,10 +84,6 @@ class InstSI <dag outs, dag ins, string asm = "",
   // Is it possible for this instruction to be atomic?
   field bit maybeAtomic = 1;
 
-  // This bit indicates that this is a VI instruction which is renamed
-  // in GFX9. Required for correct mapping from pseudo to MC.
-  field bit renamedInGFX9 = 0;
-
   // This bit indicates that this has a floating point result type, so
   // the clamp modifier has floating point semantics.
   field bit FPClamp = 0;
@@ -214,7 +210,9 @@ class InstSI <dag outs, dag ins, string asm = "",
   let TSFlags{42} = VOP3_OPSEL;
 
   let TSFlags{43} = maybeAtomic;
-  let TSFlags{44} = renamedInGFX9;
+
+  // Reserved, must be 0.
+  let TSFlags{44} = 0;
 
   let TSFlags{45} = FPClamp;
   let TSFlags{46} = IntClamp;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9ad0b4c65e1d9..44ee5c56a237b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -9189,13 +9189,43 @@ bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const {
   }
 }
 
+#define GENERATE_RENAMED_GFX9_CASES(OPCODE)                                    \
+  case OPCODE##_dpp:                                                           \
+  case OPCODE##_e32:                                                           \
+  case OPCODE##_e64:                                                           \
+  case OPCODE##_e64_dpp:                                                       \
+  case OPCODE##_sdwa:
+
+static bool isRenamedInGFX9(int Opcode) {
+  switch (Opcode) {
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADDC_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_CO_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_ADD_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBBREV_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBB_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_CO_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUBREV_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_CO_U32)
+    GENERATE_RENAMED_GFX9_CASES(AMDGPU::V_SUB_U32)
+  //
+  case AMDGPU::V_DIV_FIXUP_F16_gfx9_e64:
+  case AMDGPU::V_FMA_F16_gfx9_e64:
+  case AMDGPU::V_INTERP_P2_F16:
+  case AMDGPU::V_MAD_F16_e64:
+  case AMDGPU::V_MAD_U16_e64:
+  case AMDGPU::V_MAD_I16_e64:
+    return true;
+  default:
+    return false;
+  }
+}
+
 int SIInstrInfo::pseudoToMCOpcode(int Opcode) const {
   Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Opcode);
 
   unsigned Gen = subtargetEncodingFamily(ST);
 
-  if ((get(Opcode).TSFlags & SIInstrFlags::renamedInGFX9) != 0 &&
-    ST.getGeneration() == AMDGPUSubtarget::GFX9)
+  if (ST.getGeneration() == AMDGPUSubtarget::GFX9 && isRenamedInGFX9(Opcode))
     Gen = SIEncodingFamily::GFX9;
 
   // Adjust the encoding family to GFX80 for D16 buffer instructions when the
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 2237b2e78c417..f59d29bd81403 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -287,8 +287,14 @@ void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
   // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
   // We never need to allocate a spill for these because we don't even need to
   // restore the inactive lanes for them (they're scratchier than the usual
-  // scratch registers).
-  if (isChainFunction() && SIRegisterInfo::isChainScratchRegister(VGPR))
+  // scratch registers). We only need to do this if we have calls to
+  // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
+  // chain functions do not return) and the function did not contain a call to
+  // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
+  // when entering the function).
+  if (isChainFunction() &&
+      (SIRegisterInfo::isChainScratchRegister(VGPR) ||
+       !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))
     return;
 
   WWMSpills.insert(std::make_pair(
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
index 5f7549c2921ed..a8e4ce133ffbc 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
@@ -645,14 +645,18 @@ unsigned const DfmtNfmt2UFmtGFX11[] = {
 
 namespace Swizzle {
 
+// clang-format off
 // This must be in sync with llvm::AMDGPU::Swizzle::Id enum members, see SIDefines.h.
-const char* const IdSymbolic[] = {
+const char *const IdSymbolic[] = {
   "QUAD_PERM",
   "BITMASK_PERM",
   "SWAP",
   "REVERSE",
   "BROADCAST",
+  "FFT",
+  "ROTATE",
 };
+// clang-format on
 
 } // namespace Swizzle
 
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 52f7be3b4577d..02ec722296e7e 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -142,26 +142,21 @@ class getVOP2Pat64 <SDPatternOperator node, VOPProfile P> : LetDummies {
 multiclass VOP2Inst_e32<string opName,
                         VOPProfile P,
                         SDPatternOperator node = null_frag,
-                        string revOp = opName,
-                        bit GFX9Renamed = 0> {
-  let renamedInGFX9 = GFX9Renamed in {
+                        string revOp = opName> {
     def _e32 : VOP2_Pseudo <opName, P, VOPPatOrNull<node,P>.ret>,
                Commutable_REV<revOp#"_e32", !eq(revOp, opName)>;
-  } // End renamedInGFX9 = GFX9Renamed
 }
 multiclass
     VOP2Inst_e32_VOPD<string opName, VOPProfile P, bits<5> VOPDOp,
                       string VOPDName, SDPatternOperator node = null_frag,
-                      string revOp = opName, bit GFX9Renamed = 0> {
-  defm NAME : VOP2Inst_e32<opName, P, node, revOp, GFX9Renamed>,
+                      string revOp = opName> {
+  defm NAME : VOP2Inst_e32<opName, P, node, revOp>,
               VOPD_Component<VOPDOp, VOPDName>;
 }
 multiclass VOP2Inst_e64<string opName,
                         VOPProfile P,
                         SDPatternOperator node = null_frag,
-                        string revOp = opName,
-                        bit GFX9Renamed = 0> {
-  let renamedInGFX9 = GFX9Renamed in {
+                        string revOp = opName> {
     def _e64 : VOP3InstBase <opName, P, node, 1>,
                Commutable_REV<revOp#"_e64", !eq(revOp, opName)>;
 
@@ -169,45 +164,37 @@ multiclass VOP2Inst_e64<string opName,
       if P.HasExtVOP3DPP then
         def _e64_dpp  : VOP3_DPP_Pseudo <opName, P>;
     } // End SubtargetPredicate = isGFX11Plus
-  } // End renamedInGFX9 = GFX9Renamed
 }
 
 multiclass VOP2Inst_sdwa<string opName,
-                         VOPProfile P,
-                         bit GFX9Renamed = 0> {
-  let renamedInGFX9 = GFX9Renamed in {
+                         VOPProfile P> {
     if P.HasExtSDWA then
       def _sdwa : VOP2_SDWA_Pseudo <opName, P>;
-  } // End renamedInGFX9 = GFX9Renamed
 }
 
 multiclass VOP2Inst<string opName,
                     VOPProfile P,
                     SDPatternOperator node = null_frag,
-                    string revOp = opName,
-                    bit GFX9Renamed = 0> :
-    VOP2Inst_e32<opName, P, node, revOp, GFX9Renamed>,
-    VOP2Inst_e64<opName, P, node, revOp, GFX9Renamed>,
-    VOP2Inst_sdwa<opName, P, GFX9Renamed> {
-  let renamedInGFX9 = GFX9Renamed in {
+                    string revOp = opName> :
+    VOP2Inst_e32<opName, P, node, revOp>,
+    VOP2Inst_e64<opName, P, node, revOp>,
+    VOP2Inst_sdwa<opName, P> {
     if P.HasExtDPP then
       def _dpp  : VOP2_DPP_Pseudo <opName, P>;
-  }
 }
 
 multiclass VOP2Inst_t16<string opName,
                         VOPProfile P,
                         SDPatternOperator node = null_frag,
-                        string revOp = opName,
-                        bit GFX9Renamed = 0> {
+                        string revOp = opName> {
   let OtherPredicates = [Has16BitInsts], True16Predicate = NotHasTrue16BitInsts in {
-    defm NAME : VOP2Inst<opName, P, node, revOp, GFX9Renamed>;
+    defm NAME : VOP2Inst<opName, P, node, revOp>;
   }
   let SubtargetPredicate = UseRealTrue16Insts in {
-    defm _t16 : VOP2Inst<opName#"_t16", VOPProfile_True16<P>, node, revOp#"_t16", GFX9Renamed>;
+    defm _t16 : VOP2Inst<opName#"_t16", VOPProfile_True16<P>, node, revOp#"_t16">;
   }
   let SubtargetPredicate = UseFakeTrue16Insts in {
-    defm _fake16 : VOP2Inst<opName#"_fake16", VOPProfile_Fake16<P>, node, revOp#"_fake16", GFX9Renamed>;
+    defm _fake16 : VOP2Inst<opName#"_fake16", VOPProfile_Fake16<P>, node, revOp#"_fake16">;
   }
 }
 
@@ -218,13 +205,12 @@ multiclass VOP2Inst_t16<string opName,
 multiclass VOP2Inst_e64_t16<string opName,
                         VOPProfile P,
                         SDPatternOperator node = null_frag,
-                        string revOp = opName,
-                        bit GFX9Renamed = 0> {
+                        string revOp = opName> {
   let OtherPredicates = [Has16BitInsts], True16Predicate = NotHasTrue16BitInsts in {
-    defm NAME : VOP2Inst<opName, P, node, revOp, GFX9Renamed>;
+    defm NAME : VOP2Inst<opName, P, node, revOp>;
   }
   let SubtargetPredicate = HasTrue16BitInsts in {
-    defm _t16 : VOP2Inst_e64<opName#"_t16", VOPProfile_Fake16<P>, node, revOp#"_t16", GFX9Renamed>;
+    defm _t16 : VOP2Inst_e64<opName#"_t16", VOPProfile_Fake16<P>, node, revOp#"_t16">;
   }
 }
 
@@ -233,24 +219,19 @@ multiclass VOP2Inst_VOPD<string opName,
                          bits<5> VOPDOp,
                          string VOPDName,
                          SDPatternOperator node = null_frag,
-                         string revOp = opName,
-                         bit GFX9Renamed = 0> :
-    VOP2Inst_e32_VOPD<opName, P, VOPDOp, VOPDName, node, revOp, GFX9Renamed>,
-    VOP2Inst_e64<opName, P, node, revOp, GFX9Renamed>,
-    VOP2Inst_sdwa<opName, P, GFX9Renamed> {
-  let renamedInGFX9 = GFX9Renamed in {
+                         string revOp = opName> :
+    VOP2Inst_e32_VOPD<opName, P, VOPDOp, VOPDName, node, revOp>,
+    VOP2Inst_e64<opName, P, node, revOp>,
+    VOP2Inst_sdwa<opName, P> {
     if P.HasExtDPP then
       def _dpp  : VOP2_DPP_Pseudo <opName, P>;
-  }
 }
 
 multiclass VOP2bInst <string opName,
                       VOPProfile P,
                       SDPatternOperator node = null_frag,
                       string revOp = opName,
-                      bit GFX9Renamed = 0,
                       bit useSGPRInput = !eq(P.NumSrcArgs, 3)> {
-  let renamedInGFX9 = GFX9Renamed in {
     let SchedRW = [Write32Bit, WriteSALU] in {
       let Uses = !if(useSGPRInput, [VCC, EXEC], [EXEC]), Defs = [VCC] in {
         def _e32 : VOP2_Pseudo <opName, P, VOPPatOrNull<node,P>.ret>,
@@ -274,7 +255,6 @@ multiclass VOP2bInst <string opName,
           def _e64_dpp  : VOP3_DPP_Pseudo <opName, P>;
       } // End SubtargetPredicate = isGFX11Plus
     }
-  }
 }
 
 class VOP2bInstAlias <VOP2_Pseudo ps, Instruction inst,
@@ -763,26 +743,24 @@ def V_MADAK_F32 : VOP2_Pseudo <"v_madak_f32", VOP_MADAK_F32, []>;
 
 // No patterns so that the scalar instructions are always selected.
 // The scalar versions will be replaced with vector when needed later.
+defm V_SUB_CO_U32 : VOP2bInst <"v_sub_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_sub_co_u32">;
+defm V_SUBREV_CO_U32 : VOP2bInst <"v_subrev_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_sub_co_u32">;
+defm V_SUBB_U32 : VOP2bInst <"v_subb_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_subb_u32">;
+defm V_SUBBREV_U32 : VOP2bInst <"v_subbrev_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_subb_u32">;
 
-let isAdd = 1 in {
-  defm V_ADD_CO_U32 : VOP2bInst <"v_add_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_add_co_u32", 1>;
-}
-
-defm V_SUB_CO_U32 : VOP2bInst <"v_sub_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_sub_co_u32", 1>;
-defm V_SUBREV_CO_U32 : VOP2bInst <"v_subrev_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_sub_co_u32", 1>;
-defm V_ADDC_U32 : VOP2bInst <"v_addc_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_addc_u32", 1>;
-defm V_SUBB_U32 : VOP2bInst <"v_subb_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_subb_u32", 1>;
-defm V_SUBBREV_U32 : VOP2bInst <"v_subbrev_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_subb_u32", 1>;
 
+let SubtargetPredicate = HasAddNoCarryInsts, isReMaterializable = 1 in { 
+  defm V_SUB_U32 : VOP2Inst <"v_sub_u32", VOP_I32_I32_I32_ARITH, null_frag, "v_sub_u32">;
+  defm V_SUBREV_U32 : VOP2Inst <"v_subrev_u32", VOP_I32_I32_I32_ARITH, null_frag, "v_sub_u32">;
+}
 
-let SubtargetPredicate = HasAddNoCarryInsts, isReMaterializable = 1 in {
-
-let isAdd = 1 in {
-  defm V_ADD_U32 : VOP2Inst_VOPD <"v_add_u32", VOP_I32_I32_I32_ARITH, 0x10, "v_add_nc_u32", null_frag, "v_add_u32", 1>;
+let SubtargetPredicate = HasAddNoCarryInsts, isReMaterializable = 1, isAdd = 1 in { 
+  defm V_ADD_U32 : VOP2Inst_VOPD <"v_add_u32", VOP_I32_I32_I32_ARITH, 0x10, "v_add_nc_u32", null_frag, "v_add_u32">;
 }
 
-defm V_SUB_U32 : VOP2Inst <"v_sub_u32", VOP_I32_I32_I32_ARITH, null_frag, "v_sub_u32", 1>;
-defm V_SUBREV_U32 : VOP2Inst <"v_subrev_u32", VOP_I32_I32_I32_ARITH, null_frag, "v_sub_u32", 1>;
+let isAdd = 1 in { 
+  defm V_ADD_CO_U32 : VOP2bInst <"v_add_co_u32", VOP2b_I32_I1_I32_I32, null_frag, "v_add_co_u32">;
+  defm V_ADDC_U32 : VOP2bInst <"v_addc_u32", VOP2b_I32_I1_I32_I32_I1, null_frag, "v_addc_u32">;
 }
 
 } // End isCommutable = 1
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 20beb41b7b58b..2309ae6bf158c 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -335,35 +335,33 @@ let FPDPRounding = 1 in {
     defm V_FMA_F16 : VOP3Inst <"v_fma_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, any_fma>;
   } // End Predicates = [Has16BitInsts, isGFX8Only]
 
-  let renamedInGFX9 = 1, SubtargetPredicate = isGFX9Plus in {
+  let SubtargetPredicate = isGFX9Plus in {
     defm V_DIV_FIXUP_F16_gfx9 : VOP3Inst <"v_div_fixup_f16_gfx9",
                                           VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUdiv_fixup>;
     defm V_FMA_F16_gfx9 : VOP3Inst <"v_fma_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, any_fma>;
-  } // End renamedInGFX9 = 1, SubtargetPredicate = isGFX9Plus
+  } // End SubtargetPredicate = isGFX9Plus
 } // End FPDPRounding = 1
 
 let SubtargetPredicate = Has16BitInsts, isCommutable = 1 in {
 
-let renamedInGFX9 = 1 in {
-  defm V_MAD_U16 : VOP3Inst <"v_mad_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
-  defm V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
-  let FPDPRounding = 1 in {
-    defm V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, any_fmad>;
-    let Uses = [MODE, M0, EXEC] in {
-    let OtherPredicates = [isNotGFX90APlus] in
-    // For some reason the intrinsic operands are in a different order
-    // from the instruction operands.
-    def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>,
-           [(set f16:$vdst,
-             (int_amdgcn_interp_p2_f16 (VOP3Mods f32:$src2, i32:$src2_modifiers),
-                                       (VOP3Mods f32:$src0, i32:$src0_modifiers),
-                                       (i32 timm:$attrchan),
-                                       (i32 timm:$attr),
-                                       (i1 timm:$high),
-                                       M0))]>;
-    } // End Uses = [M0, MODE, EXEC]
-  } // End FPDPRounding = 1
-} // End renamedInGFX9 = 1
+defm V_MAD_U16 : VOP3Inst <"v_mad_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
+defm V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CLAMP>>;
+let FPDPRounding = 1 in {
+  defm V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, any_fmad>;
+  let Uses = [MODE, M0, EXEC] in {
+  let OtherPredicates = [isNotGFX90APlus] in
+  // For some reason the intrinsic operands are in a different order
+  // from the instruction operands.
+  def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>,
+          [(set f16:$vdst,
+            (int_amdgcn_interp_p2_f16 (VOP3Mods f32:$src2, i32:$src2_modifiers),
+                                      (VOP3Mods f32:$src0, i32:$src0_modifiers),
+                                      (i32 timm:$attrchan),
+                                      (i32 timm:$attr),
+                                      (i1 timm:$high),
+                                      M0))]>;
+  } // End Uses = [M0, MODE, EXEC]
+} // End FPDPRounding = 1
 
 let SubtargetPredicate = isGFX9Only, FPDPRounding = 1 in {
   defm V_MAD_F16_gfx9   : VOP3Inst <"v_mad_f16_gfx9", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>> ;
diff --git a/llvm/lib/Target/ARM/ARMFeatures.td b/llvm/lib/Target/ARM/ARMFeatures.td
index dc0e86c696f63..c1449adc34dc7 100644
--- a/llvm/lib/Target/ARM/ARMFeatures.td
+++ b/llvm/lib/Target/ARM/ARMFeatures.td
@@ -372,11 +372,11 @@ def FeatureVMLxForwarding : SubtargetFeature<"vmlx-forwarding",
 def FeaturePref32BitThumb : SubtargetFeature<"32bit", "Prefers32BitThumb", "true",
                                              "Prefer 32-bit Thumb instrs">;
 
-def FeaturePrefLoopAlign32 : SubtargetFeature<"loop-align", "PrefLoopLogAlignment","2",
-                                              "Prefer 32-bit alignment for loops">;
+def FeaturePreferBranchAlign32 : SubtargetFeature<"loop-align", "PreferBranchLogAlignment","2",
+                                              "Prefer 32-bit alignment for branch targets">;
 
-def FeaturePrefLoopAlign64 : SubtargetFeature<"loop-align-64", "PrefLoopLogAlignment","3",
-                                              "Prefer 64-bit alignment for loops">;
+def FeaturePreferBranchAlign64 : SubtargetFeature<"branch-align-64", "PreferBranchLogAlignment","3",
+                                              "Prefer 64-bit alignment for branch targets">;
 
 def FeatureMVEVectorCostFactor1 : SubtargetFeature<"mve1beat", "MVEVectorCostFactor", "4",
                         "Model MVE instructions as a 1 beat per tick architecture">;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index f891aece26848..1733424a8b669 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -1635,8 +1635,9 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   // Prefer likely predicted branches to selects on out-of-order cores.
   PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
 
-  setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
-  setPrefFunctionAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
+  setPrefLoopAlignment(Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
+  setPrefFunctionAlignment(
+      Align(1ULL << Subtarget->getPreferBranchLogAlignment()));
 
   setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
 }
diff --git a/llvm/lib/Target/ARM/ARMProcessors.td b/llvm/lib/Target/ARM/ARMProcessors.td
index a66a2c0b1981d..ce767b2b968e1 100644
--- a/llvm/lib/Target/ARM/ARMProcessors.td
+++ b/llvm/lib/Target/ARM/ARMProcessors.td
@@ -324,7 +324,7 @@ def : ProcessorModel<"cortex-r8",   CortexA8Model,      [ARMv7r,
 
 def : ProcessorModel<"cortex-m3",   CortexM4Model,      [ARMv7m,
                                                          ProcM3,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureUseMISched,
                                                          FeatureHasNoBranchPredictor]>;
 
@@ -335,7 +335,7 @@ def : ProcessorModel<"sc300",       CortexM4Model,      [ARMv7m,
 
 def : ProcessorModel<"cortex-m4", CortexM4Model,        [ARMv7em,
                                                          FeatureVFP4_D16_SP,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureHasSlowFPVMLx,
                                                          FeatureHasSlowFPVFMx,
                                                          FeatureUseMISched,
@@ -344,7 +344,7 @@ def : ProcessorModel<"cortex-m4", CortexM4Model,        [ARMv7em,
 def : ProcessorModel<"cortex-m7", CortexM7Model,        [ARMv7em,
                                                          ProcM7,
                                                          FeatureFPARMv8_D16,
-                                                         FeaturePrefLoopAlign64,
+                                                         FeaturePreferBranchAlign64,
                                                          FeatureUseMIPipeliner,
                                                          FeatureUseMISched]>;
 
@@ -355,7 +355,7 @@ def : ProcNoItin<"cortex-m23",                          [ARMv8mBaseline,
 def : ProcessorModel<"cortex-m33", CortexM4Model,       [ARMv8mMainline,
                                                          FeatureDSP,
                                                          FeatureFPARMv8_D16_SP,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureHasSlowFPVMLx,
                                                          FeatureHasSlowFPVFMx,
                                                          FeatureUseMISched,
@@ -365,7 +365,7 @@ def : ProcessorModel<"cortex-m33", CortexM4Model,       [ARMv8mMainline,
 def : ProcessorModel<"cortex-m35p", CortexM4Model,      [ARMv8mMainline,
                                                          FeatureDSP,
                                                          FeatureFPARMv8_D16_SP,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureHasSlowFPVMLx,
                                                          FeatureHasSlowFPVFMx,
                                                          FeatureUseMISched,
@@ -377,7 +377,7 @@ def : ProcessorModel<"cortex-m55", CortexM55Model,      [ARMv81mMainline,
                                                          FeatureFPARMv8_D16,
                                                          FeatureUseMISched,
                                                          FeatureHasNoBranchPredictor,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureHasSlowFPVMLx,
                                                          HasMVEFloatOps,
                                                          FeatureFixCMSE_CVE_2021_35465]>;
@@ -386,7 +386,7 @@ def : ProcessorModel<"cortex-m85", CortexM85Model,      [ARMv81mMainline,
                                                          FeatureDSP,
                                                          FeatureFPARMv8_D16,
                                                          FeaturePACBTI,
-                                                         FeaturePrefLoopAlign64,
+                                                         FeaturePreferBranchAlign64,
                                                          FeatureUseMISched,
                                                          HasMVEFloatOps]>;
 
@@ -396,7 +396,7 @@ def : ProcessorModel<"cortex-m52", CortexM55Model,      [ARMv81mMainline,
                                                          FeatureHasNoBranchPredictor,
                                                          FeaturePACBTI,
                                                          FeatureUseMISched,
-                                                         FeaturePrefLoopAlign32,
+                                                         FeaturePreferBranchAlign32,
                                                          FeatureHasSlowFPVMLx,
                                                          FeatureMVEVectorCostFactor1,
                                                          HasMVEFloatOps]>;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 13018e647e822..f9d822873bb00 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -302,7 +302,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
     LdStMultipleTiming = SingleIssuePlusExtras;
     MaxInterleaveFactor = 4;
     if (!isThumb())
-      PrefLoopLogAlignment = 3;
+      PreferBranchLogAlignment = 3;
     break;
   case Kryo:
     break;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index fa20f4b590bea..1ca5bd4a620c4 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -133,7 +133,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   int PreISelOperandLatencyAdjustment = 2;
 
   /// What alignment is preferred for loop bodies and functions, in log2(bytes).
-  unsigned PrefLoopLogAlignment = 0;
+  unsigned PreferBranchLogAlignment = 0;
 
   /// The cost factor for MVE instructions, representing the multiple beats an
   // instruction can take. The default is 2, (set in initSubtargetFeatures so
@@ -476,7 +476,9 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
     return isROPI() || !isTargetELF();
   }
 
-  unsigned getPrefLoopLogAlignment() const { return PrefLoopLogAlignment; }
+  unsigned getPreferBranchLogAlignment() const {
+    return PreferBranchLogAlignment;
+  }
 
   unsigned
   getMVEVectorCostFactor(TargetTransformInfo::TargetCostKind CostKind) const {
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index 2b78ed7134c92..660e00b893c88 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -440,7 +440,7 @@ static Value *aspaceWrapValue(DenseMap<Value *, Value *> &Cache, Function *F,
     auto *GEPTy = cast<PointerType>(GEP->getType());
     auto *NewGEP = GEP->clone();
     NewGEP->insertAfter(GEP);
-    NewGEP->mutateType(GEPTy->getPointerTo(0));
+    NewGEP->mutateType(PointerType::getUnqual(GEPTy->getContext()));
     NewGEP->setOperand(GEP->getPointerOperandIndex(), WrappedPtr);
     NewGEP->setName(GEP->getName());
     Cache[ToWrap] = NewGEP;
@@ -452,8 +452,7 @@ static Value *aspaceWrapValue(DenseMap<Value *, Value *> &Cache, Function *F,
     IB.SetInsertPoint(*InsnPtr->getInsertionPointAfterDef());
   else
     IB.SetInsertPoint(F->getEntryBlock().getFirstInsertionPt());
-  auto *PtrTy = cast<PointerType>(ToWrap->getType());
-  auto *ASZeroPtrTy = PtrTy->getPointerTo(0);
+  auto *ASZeroPtrTy = IB.getPtrTy(0);
   auto *ACast = IB.CreateAddrSpaceCast(ToWrap, ASZeroPtrTy, ToWrap->getName());
   Cache[ToWrap] = ACast;
   return ACast;
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index 7e0f8a145505e..c8ef0ef6f7e70 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_target(DirectXCodeGen
   DirectXTargetMachine.cpp
   DirectXTargetTransformInfo.cpp
   DXContainerGlobals.cpp
+  DXILDataScalarization.cpp
   DXILFinalizeLinkage.cpp
   DXILIntrinsicExpansion.cpp
   DXILOpBuilder.cpp
diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
new file mode 100644
index 0000000000000..0e6cf59e25750
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -0,0 +1,300 @@
+//===- DXILDataScalarization.cpp - Perform DXIL Data Legalization ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#include "DXILDataScalarization.h"
+#include "DirectX.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/DXILResource.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ReplaceConstant.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/Local.h"
+
+#define DEBUG_TYPE "dxil-data-scalarization"
+static const int MaxVecSize = 4;
+
+using namespace llvm;
+
+class DXILDataScalarizationLegacy : public ModulePass {
+
+public:
+  bool runOnModule(Module &M) override;
+  DXILDataScalarizationLegacy() : ModulePass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  static char ID; // Pass identification.
+};
+
+static bool findAndReplaceVectors(Module &M);
+
+class DataScalarizerVisitor : public InstVisitor<DataScalarizerVisitor, bool> {
+public:
+  DataScalarizerVisitor() : GlobalMap() {}
+  bool visit(Function &F);
+  // InstVisitor methods.  They return true if the instruction was scalarized,
+  // false if nothing changed.
+  bool visitInstruction(Instruction &I) { return false; }
+  bool visitSelectInst(SelectInst &SI) { return false; }
+  bool visitICmpInst(ICmpInst &ICI) { return false; }
+  bool visitFCmpInst(FCmpInst &FCI) { return false; }
+  bool visitUnaryOperator(UnaryOperator &UO) { return false; }
+  bool visitBinaryOperator(BinaryOperator &BO) { return false; }
+  bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
+  bool visitCastInst(CastInst &CI) { return false; }
+  bool visitBitCastInst(BitCastInst &BCI) { return false; }
+  bool visitInsertElementInst(InsertElementInst &IEI) { return false; }
+  bool visitExtractElementInst(ExtractElementInst &EEI) { return false; }
+  bool visitShuffleVectorInst(ShuffleVectorInst &SVI) { return false; }
+  bool visitPHINode(PHINode &PHI) { return false; }
+  bool visitLoadInst(LoadInst &LI);
+  bool visitStoreInst(StoreInst &SI);
+  bool visitCallInst(CallInst &ICI) { return false; }
+  bool visitFreezeInst(FreezeInst &FI) { return false; }
+  friend bool findAndReplaceVectors(llvm::Module &M);
+
+private:
+  GlobalVariable *lookupReplacementGlobal(Value *CurrOperand);
+  DenseMap<GlobalVariable *, GlobalVariable *> GlobalMap;
+  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
+  bool finish();
+};
+
+bool DataScalarizerVisitor::visit(Function &F) {
+  assert(!GlobalMap.empty());
+  ReversePostOrderTraversal<BasicBlock *> RPOT(&F.getEntryBlock());
+  for (BasicBlock *BB : RPOT) {
+    for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
+      Instruction *I = &*II;
+      bool Done = InstVisitor::visit(I);
+      ++II;
+      if (Done && I->getType()->isVoidTy())
+        I->eraseFromParent();
+    }
+  }
+  return finish();
+}
+
+bool DataScalarizerVisitor::finish() {
+  RecursivelyDeleteTriviallyDeadInstructionsPermissive(PotentiallyDeadInstrs);
+  return true;
+}
+
+GlobalVariable *
+DataScalarizerVisitor::lookupReplacementGlobal(Value *CurrOperand) {
+  if (GlobalVariable *OldGlobal = dyn_cast<GlobalVariable>(CurrOperand)) {
+    auto It = GlobalMap.find(OldGlobal);
+    if (It != GlobalMap.end()) {
+      return It->second; // Found, return the new global
+    }
+  }
+  return nullptr; // Not found
+}
+
+bool DataScalarizerVisitor::visitLoadInst(LoadInst &LI) {
+  unsigned NumOperands = LI.getNumOperands();
+  for (unsigned I = 0; I < NumOperands; ++I) {
+    Value *CurrOpperand = LI.getOperand(I);
+    if (GlobalVariable *NewGlobal = lookupReplacementGlobal(CurrOpperand))
+      LI.setOperand(I, NewGlobal);
+  }
+  return false;
+}
+
+bool DataScalarizerVisitor::visitStoreInst(StoreInst &SI) {
+  unsigned NumOperands = SI.getNumOperands();
+  for (unsigned I = 0; I < NumOperands; ++I) {
+    Value *CurrOpperand = SI.getOperand(I);
+    if (GlobalVariable *NewGlobal = lookupReplacementGlobal(CurrOpperand)) {
+      SI.setOperand(I, NewGlobal);
+    }
+  }
+  return false;
+}
+
+bool DataScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
+  unsigned NumOperands = GEPI.getNumOperands();
+  for (unsigned I = 0; I < NumOperands; ++I) {
+    Value *CurrOpperand = GEPI.getOperand(I);
+    GlobalVariable *NewGlobal = lookupReplacementGlobal(CurrOpperand);
+    if (!NewGlobal)
+      continue;
+    IRBuilder<> Builder(&GEPI);
+
+    SmallVector<Value *, MaxVecSize> Indices;
+    for (auto &Index : GEPI.indices())
+      Indices.push_back(Index);
+
+    Value *NewGEP =
+        Builder.CreateGEP(NewGlobal->getValueType(), NewGlobal, Indices);
+
+    GEPI.replaceAllUsesWith(NewGEP);
+    PotentiallyDeadInstrs.emplace_back(&GEPI);
+  }
+  return true;
+}
+
+// Recursively Creates and Array like version of the given vector like type.
+static Type *replaceVectorWithArray(Type *T, LLVMContext &Ctx) {
+  if (auto *VecTy = dyn_cast<VectorType>(T))
+    return ArrayType::get(VecTy->getElementType(),
+                          dyn_cast<FixedVectorType>(VecTy)->getNumElements());
+  if (auto *ArrayTy = dyn_cast<ArrayType>(T)) {
+    Type *NewElementType =
+        replaceVectorWithArray(ArrayTy->getElementType(), Ctx);
+    return ArrayType::get(NewElementType, ArrayTy->getNumElements());
+  }
+  // If it's not a vector or array, return the original type.
+  return T;
+}
+
+Constant *transformInitializer(Constant *Init, Type *OrigType, Type *NewType,
+                               LLVMContext &Ctx) {
+  // Handle ConstantAggregateZero (zero-initialized constants)
+  if (isa<ConstantAggregateZero>(Init)) {
+    return ConstantAggregateZero::get(NewType);
+  }
+
+  // Handle UndefValue (undefined constants)
+  if (isa<UndefValue>(Init)) {
+    return UndefValue::get(NewType);
+  }
+
+  // Handle vector to array transformation
+  if (isa<VectorType>(OrigType) && isa<ArrayType>(NewType)) {
+    // Convert vector initializer to array initializer
+    SmallVector<Constant *, MaxVecSize> ArrayElements;
+    if (ConstantVector *ConstVecInit = dyn_cast<ConstantVector>(Init)) {
+      for (unsigned I = 0; I < ConstVecInit->getNumOperands(); ++I)
+        ArrayElements.push_back(ConstVecInit->getOperand(I));
+    } else if (ConstantDataVector *ConstDataVecInit =
+                   llvm::dyn_cast<llvm::ConstantDataVector>(Init)) {
+      for (unsigned I = 0; I < ConstDataVecInit->getNumElements(); ++I)
+        ArrayElements.push_back(ConstDataVecInit->getElementAsConstant(I));
+    } else {
+      assert(false && "Expected a ConstantVector or ConstantDataVector for "
+                      "vector initializer!");
+    }
+
+    return ConstantArray::get(cast<ArrayType>(NewType), ArrayElements);
+  }
+
+  // Handle array of vectors transformation
+  if (auto *ArrayTy = dyn_cast<ArrayType>(OrigType)) {
+    auto *ArrayInit = dyn_cast<ConstantArray>(Init);
+    assert(ArrayInit && "Expected a ConstantArray for array initializer!");
+
+    SmallVector<Constant *, MaxVecSize> NewArrayElements;
+    for (unsigned I = 0; I < ArrayTy->getNumElements(); ++I) {
+      // Recursively transform array elements
+      Constant *NewElemInit = transformInitializer(
+          ArrayInit->getOperand(I), ArrayTy->getElementType(),
+          cast<ArrayType>(NewType)->getElementType(), Ctx);
+      NewArrayElements.push_back(NewElemInit);
+    }
+
+    return ConstantArray::get(cast<ArrayType>(NewType), NewArrayElements);
+  }
+
+  // If not a vector or array, return the original initializer
+  return Init;
+}
+
+static bool findAndReplaceVectors(Module &M) {
+  bool MadeChange = false;
+  LLVMContext &Ctx = M.getContext();
+  IRBuilder<> Builder(Ctx);
+  DataScalarizerVisitor Impl;
+  for (GlobalVariable &G : M.globals()) {
+    Type *OrigType = G.getValueType();
+
+    Type *NewType = replaceVectorWithArray(OrigType, Ctx);
+    if (OrigType != NewType) {
+      // Create a new global variable with the updated type
+      // Note: Initializer is set via transformInitializer
+      GlobalVariable *NewGlobal = new GlobalVariable(
+          M, NewType, G.isConstant(), G.getLinkage(),
+          /*Initializer=*/nullptr, G.getName() + ".scalarized", &G,
+          G.getThreadLocalMode(), G.getAddressSpace(),
+          G.isExternallyInitialized());
+
+      // Copy relevant attributes
+      NewGlobal->setUnnamedAddr(G.getUnnamedAddr());
+      if (G.getAlignment() > 0) {
+        NewGlobal->setAlignment(G.getAlign());
+      }
+
+      if (G.hasInitializer()) {
+        Constant *Init = G.getInitializer();
+        Constant *NewInit = transformInitializer(Init, OrigType, NewType, Ctx);
+        NewGlobal->setInitializer(NewInit);
+      }
+
+      // Note: we want to do G.replaceAllUsesWith(NewGlobal);, but it assumes
+      // type equality. Instead we will use the visitor pattern.
+      Impl.GlobalMap[&G] = NewGlobal;
+      for (User *U : make_early_inc_range(G.users())) {
+        if (isa<ConstantExpr>(U) && isa<Operator>(U)) {
+          ConstantExpr *CE = cast<ConstantExpr>(U);
+          convertUsersOfConstantsToInstructions(CE,
+                                                /*RestrictToFunc=*/nullptr,
+                                                /*RemoveDeadConstants=*/false,
+                                                /*IncludeSelf=*/true);
+        }
+        if (isa<Instruction>(U)) {
+          Instruction *Inst = cast<Instruction>(U);
+          Function *F = Inst->getFunction();
+          if (F)
+            Impl.visit(*F);
+        }
+      }
+    }
+  }
+
+  // Remove the old globals after the iteration
+  for (auto &[Old, New] : Impl.GlobalMap) {
+    Old->eraseFromParent();
+    MadeChange = true;
+  }
+  return MadeChange;
+}
+
+PreservedAnalyses DXILDataScalarization::run(Module &M,
+                                             ModuleAnalysisManager &) {
+  bool MadeChanges = findAndReplaceVectors(M);
+  if (!MadeChanges)
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserve<DXILResourceAnalysis>();
+  return PA;
+}
+
+bool DXILDataScalarizationLegacy::runOnModule(Module &M) {
+  return findAndReplaceVectors(M);
+}
+
+void DXILDataScalarizationLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addPreserved<DXILResourceWrapperPass>();
+}
+
+char DXILDataScalarizationLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(DXILDataScalarizationLegacy, DEBUG_TYPE,
+                      "DXIL Data Scalarization", false, false)
+INITIALIZE_PASS_END(DXILDataScalarizationLegacy, DEBUG_TYPE,
+                    "DXIL Data Scalarization", false, false)
+
+ModulePass *llvm::createDXILDataScalarizationLegacyPass() {
+  return new DXILDataScalarizationLegacy();
+}
diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.h b/llvm/lib/Target/DirectX/DXILDataScalarization.h
new file mode 100644
index 0000000000000..560e061db96d0
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.h
@@ -0,0 +1,25 @@
+//===- DXILDataScalarization.h - Perform DXIL Data Legalization -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_DIRECTX_DXILDATASCALARIZATION_H
+#define LLVM_TARGET_DIRECTX_DXILDATASCALARIZATION_H
+
+#include "DXILResource.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+/// A pass that transforms Vectors to Arrays
+class DXILDataScalarization : public PassInfoMixin<DXILDataScalarization> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+};
+} // namespace llvm
+
+#endif // LLVM_TARGET_DIRECTX_DXILDATASCALARIZATION_H
diff --git a/llvm/lib/Target/DirectX/DirectX.h b/llvm/lib/Target/DirectX/DirectX.h
index 60fc5094542b3..3221779be2f31 100644
--- a/llvm/lib/Target/DirectX/DirectX.h
+++ b/llvm/lib/Target/DirectX/DirectX.h
@@ -34,6 +34,12 @@ void initializeDXILIntrinsicExpansionLegacyPass(PassRegistry &);
 /// Pass to expand intrinsic operations that lack DXIL opCodes
 ModulePass *createDXILIntrinsicExpansionLegacyPass();
 
+/// Initializer for DXIL Data Scalarization Pass
+void initializeDXILDataScalarizationLegacyPass(PassRegistry &);
+
+/// Pass to scalarize llvm global data into a DXIL legal form
+ModulePass *createDXILDataScalarizationLegacyPass();
+
 /// Initializer for DXILOpLowering
 void initializeDXILOpLoweringLegacyPass(PassRegistry &);
 
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 606022a9835f0..f358215ecf373 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -46,6 +46,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   RegisterTargetMachine<DirectXTargetMachine> X(getTheDirectXTarget());
   auto *PR = PassRegistry::getPassRegistry();
   initializeDXILIntrinsicExpansionLegacyPass(*PR);
+  initializeDXILDataScalarizationLegacyPass(*PR);
   initializeScalarizerLegacyPassPass(*PR);
   initializeDXILPrepareModulePass(*PR);
   initializeEmbedDXILPassPass(*PR);
@@ -86,6 +87,7 @@ class DirectXPassConfig : public TargetPassConfig {
   FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
   void addCodeGenPrepare() override {
     addPass(createDXILIntrinsicExpansionLegacyPass());
+    addPass(createDXILDataScalarizationLegacyPass());
     ScalarizerPassOptions DxilScalarOptions;
     DxilScalarOptions.ScalarizeLoadStore = true;
     addPass(createScalarizerPass(DxilScalarOptions));
diff --git a/llvm/lib/Target/Lanai/LanaiRegisterInfo.h b/llvm/lib/Target/Lanai/LanaiRegisterInfo.h
index 5168dddd93019..4ff74c5f4eb1e 100644
--- a/llvm/lib/Target/Lanai/LanaiRegisterInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiRegisterInfo.h
@@ -43,8 +43,6 @@ struct LanaiRegisterInfo : public LanaiGenRegisterInfo {
   Register getFrameRegister(const MachineFunction &MF) const override;
   Register getBaseRegister() const;
   bool hasBasePointer(const MachineFunction &MF) const;
-
-  int getDwarfRegNum(unsigned RegNum, bool IsEH) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
index e267a6d0844c6..f4af1d08dde5d 100644
--- a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -81,12 +81,8 @@ bool MipsAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
   MipsFI = MF.getInfo<MipsFunctionInfo>();
   if (Subtarget->inMips16Mode())
-    for (const auto &I : MipsFI->StubsNeeded) {
-      const char *Symbol = I.first;
-      const Mips16HardFloatInfo::FuncSignature *Signature = I.second;
-      if (StubsNeeded.find(Symbol) == StubsNeeded.end())
-        StubsNeeded[Symbol] = Signature;
-    }
+    for (const auto &I : MipsFI->StubsNeeded)
+      StubsNeeded.insert(I);
   MCP = MF.getConstantPool();
 
   // In NaCl, all indirect jump targets must be aligned to bundle size.
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 9bcc911b6c345..09d84d41a2294 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -490,6 +490,7 @@ void NVPTXAsmPrinter::emitFunctionEntryLabel() {
   // Emit open brace for function body.
   OutStreamer->emitRawText(StringRef("{\n"));
   setAndEmitFunctionVirtualRegisters(*MF);
+  encodeDebugInfoRegisterNumbers(*MF);
   // Emit initial .loc debug directive for correct relocation symbol data.
   if (const DISubprogram *SP = MF->getFunction().getSubprogram()) {
     assert(SP->getUnit());
@@ -573,9 +574,30 @@ void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
   // filter it out for lower SM versions, as it causes a hard ptxas crash.
   const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
   const auto *STI = static_cast<const NVPTXSubtarget *>(NTM.getSubtargetImpl());
-  if (STI->getSmVersion() >= 90)
+
+  if (STI->getSmVersion() >= 90) {
+    std::optional<unsigned> ClusterX = getClusterDimx(F);
+    std::optional<unsigned> ClusterY = getClusterDimy(F);
+    std::optional<unsigned> ClusterZ = getClusterDimz(F);
+
+    if (ClusterX || ClusterY || ClusterZ) {
+      O << ".explicitcluster\n";
+      if (ClusterX.value_or(1) != 0) {
+        assert(ClusterY.value_or(1) && ClusterZ.value_or(1) &&
+               "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z "
+               "should be non-zero as well");
+
+        O << ".reqnctapercluster " << ClusterX.value_or(1) << ", "
+          << ClusterY.value_or(1) << ", " << ClusterZ.value_or(1) << "\n";
+      } else {
+        assert(!ClusterY.value_or(1) && !ClusterZ.value_or(1) &&
+               "cluster_dim_x == 0 implies cluster_dim_y and cluster_dim_z "
+               "should be 0 as well");
+      }
+    }
     if (const auto Maxclusterrank = getMaxClusterRank(F))
       O << ".maxclusterrank " << *Maxclusterrank << "\n";
+  }
 }
 
 std::string NVPTXAsmPrinter::getVirtualRegisterName(unsigned Reg) const {
@@ -1771,6 +1793,26 @@ void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
   OutStreamer->emitRawText(O.str());
 }
 
+/// Translate virtual register numbers in DebugInfo locations to their printed
+/// encodings, as used by CUDA-GDB.
+void NVPTXAsmPrinter::encodeDebugInfoRegisterNumbers(
+    const MachineFunction &MF) {
+  const NVPTXSubtarget &STI = MF.getSubtarget<NVPTXSubtarget>();
+  const NVPTXRegisterInfo *registerInfo = STI.getRegisterInfo();
+
+  // Clear the old mapping, and add the new one.  This mapping is used after the
+  // printing of the current function is complete, but before the next function
+  // is printed.
+  registerInfo->clearDebugRegisterMap();
+
+  for (auto &classMap : VRegMapping) {
+    for (auto &registerMapping : classMap.getSecond()) {
+      auto reg = registerMapping.getFirst();
+      registerInfo->addToDebugRegisterMap(reg, getVirtualRegisterName(reg));
+    }
+  }
+}
+
 void NVPTXAsmPrinter::printFPConstant(const ConstantFP *Fp, raw_ostream &O) {
   APFloat APF = APFloat(Fp->getValueAPF()); // make a copy
   bool ignored;
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
index d950047dc92c7..f58b4bdc40474 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -181,6 +181,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXAsmPrinter : public AsmPrinter {
   void emitVirtualRegister(unsigned int vr, raw_ostream &);
   void emitFunctionParamList(const Function *, raw_ostream &O);
   void setAndEmitFunctionVirtualRegisters(const MachineFunction &MF);
+  void encodeDebugInfoRegisterNumbers(const MachineFunction &MF);
   void printReturnValStr(const Function *, raw_ostream &O);
   void printReturnValStr(const MachineFunction &MF, raw_ostream &O);
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 2688834221091..8718b7890bf58 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -594,20 +594,13 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
   setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
 
-  // TODO: we may consider expanding ROTL/ROTR on older GPUs.  Currently on GPUs
-  // that don't have h/w rotation we lower them to multi-instruction assembly.
-  // See ROT*_sw in NVPTXIntrInfo.td
-  setOperationAction(ISD::ROTL, MVT::i64, Legal);
-  setOperationAction(ISD::ROTR, MVT::i64, Legal);
-  setOperationAction(ISD::ROTL, MVT::i32, Legal);
-  setOperationAction(ISD::ROTR, MVT::i32, Legal);
-
-  setOperationAction(ISD::ROTL, MVT::i16, Expand);
-  setOperationAction(ISD::ROTL, MVT::v2i16, Expand);
-  setOperationAction(ISD::ROTR, MVT::i16, Expand);
-  setOperationAction(ISD::ROTR, MVT::v2i16, Expand);
-  setOperationAction(ISD::ROTL, MVT::i8, Expand);
-  setOperationAction(ISD::ROTR, MVT::i8, Expand);
+  setOperationAction({ISD::ROTL, ISD::ROTR},
+                     {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},
+                     Expand);
+
+  if (STI.hasHWROT32())
+    setOperationAction({ISD::FSHL, ISD::FSHR}, MVT::i32, Legal);
+
   setOperationAction(ISD::BSWAP, MVT::i16, Expand);
 
   setOperationAction(ISD::BR_JT, MVT::Other, Custom);
@@ -958,8 +951,8 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(NVPTXISD::LDUV4)
     MAKE_CASE(NVPTXISD::StoreV2)
     MAKE_CASE(NVPTXISD::StoreV4)
-    MAKE_CASE(NVPTXISD::FUN_SHFL_CLAMP)
-    MAKE_CASE(NVPTXISD::FUN_SHFR_CLAMP)
+    MAKE_CASE(NVPTXISD::FSHL_CLAMP)
+    MAKE_CASE(NVPTXISD::FSHR_CLAMP)
     MAKE_CASE(NVPTXISD::IMAD)
     MAKE_CASE(NVPTXISD::BFE)
     MAKE_CASE(NVPTXISD::BFI)
@@ -2490,8 +2483,8 @@ SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op,
     //   dLo = shf.r.clamp aLo, aHi, Amt
 
     SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
-    SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi,
-                             ShAmt);
+    SDValue Lo =
+        DAG.getNode(NVPTXISD::FSHR_CLAMP, dl, VT, ShOpHi, ShOpLo, ShAmt);
 
     SDValue Ops[2] = { Lo, Hi };
     return DAG.getMergeValues(Ops, dl);
@@ -2549,8 +2542,8 @@ SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op,
     //   dHi = shf.l.clamp aLo, aHi, Amt
     //   dLo = aLo << Amt
 
-    SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi,
-                             ShAmt);
+    SDValue Hi =
+        DAG.getNode(NVPTXISD::FSHL_CLAMP, dl, VT, ShOpHi, ShOpLo, ShAmt);
     SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
 
     SDValue Ops[2] = { Lo, Hi };
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 70e16eee346aa..8c3a597ce0b08 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -51,8 +51,8 @@ enum NodeType : unsigned {
   CallSeqEnd,
   CallPrototype,
   ProxyReg,
-  FUN_SHFL_CLAMP,
-  FUN_SHFR_CLAMP,
+  FSHL_CLAMP,
+  FSHR_CLAMP,
   MUL_WIDE_SIGNED,
   MUL_WIDE_UNSIGNED,
   IMAD,
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 510e4b8100311..0d9dd1b8ee70a 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1665,167 +1665,6 @@ def BREV64 :
              "brev.b64 \t$dst, $a;",
              [(set Int64Regs:$dst, (bitreverse Int64Regs:$a))]>;
 
-//
-// Rotate: Use ptx shf instruction if available.
-//
-
-// 32 bit r2 = rotl r1, n
-//    =>
-//        r2 = shf.l r1, r1, n
-def ROTL32imm_hw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, i32imm:$amt),
-            "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
-            [(set Int32Regs:$dst, (rotl (i32 Int32Regs:$src), (i32 imm:$amt)))]>,
-           Requires<[hasHWROT32]>;
-
-def ROTL32reg_hw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, Int32Regs:$amt),
-            "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
-            [(set Int32Regs:$dst, (rotl (i32 Int32Regs:$src), (i32 Int32Regs:$amt)))]>,
-           Requires<[hasHWROT32]>;
-
-// 32 bit r2 = rotr r1, n
-//    =>
-//        r2 = shf.r r1, r1, n
-def ROTR32imm_hw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, i32imm:$amt),
-            "shf.r.wrap.b32 \t$dst, $src, $src, $amt;",
-            [(set Int32Regs:$dst, (rotr (i32 Int32Regs:$src), (i32 imm:$amt)))]>,
-           Requires<[hasHWROT32]>;
-
-def ROTR32reg_hw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, Int32Regs:$amt),
-            "shf.r.wrap.b32 \t$dst, $src, $src, $amt;",
-            [(set Int32Regs:$dst, (rotr (i32 Int32Regs:$src), (i32 Int32Regs:$amt)))]>,
-           Requires<[hasHWROT32]>;
-
-// 32-bit software rotate by immediate.  $amt2 should equal 32 - $amt1.
-def ROT32imm_sw :
-  NVPTXInst<(outs Int32Regs:$dst),
-            (ins Int32Regs:$src, i32imm:$amt1, i32imm:$amt2),
-            "{{\n\t"
-            ".reg .b32 %lhs;\n\t"
-            ".reg .b32 %rhs;\n\t"
-            "shl.b32 \t%lhs, $src, $amt1;\n\t"
-            "shr.b32 \t%rhs, $src, $amt2;\n\t"
-            "add.u32 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            []>;
-
-def SUB_FRM_32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(32 - N->getZExtValue(), SDLoc(N), MVT::i32);
-}]>;
-
-def : Pat<(rotl (i32 Int32Regs:$src), (i32 imm:$amt)),
-          (ROT32imm_sw Int32Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
-      Requires<[noHWROT32]>;
-def : Pat<(rotr (i32 Int32Regs:$src), (i32 imm:$amt)),
-          (ROT32imm_sw Int32Regs:$src, (SUB_FRM_32 node:$amt), imm:$amt)>,
-      Requires<[noHWROT32]>;
-
-// 32-bit software rotate left by register.
-def ROTL32reg_sw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, Int32Regs:$amt),
-            "{{\n\t"
-            ".reg .b32 %lhs;\n\t"
-            ".reg .b32 %rhs;\n\t"
-            ".reg .b32 %amt2;\n\t"
-            "shl.b32 \t%lhs, $src, $amt;\n\t"
-            "sub.s32 \t%amt2, 32, $amt;\n\t"
-            "shr.b32 \t%rhs, $src, %amt2;\n\t"
-            "add.u32 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            [(set Int32Regs:$dst, (rotl (i32 Int32Regs:$src), (i32 Int32Regs:$amt)))]>,
-           Requires<[noHWROT32]>;
-
-// 32-bit software rotate right by register.
-def ROTR32reg_sw :
-  NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, Int32Regs:$amt),
-            "{{\n\t"
-            ".reg .b32 %lhs;\n\t"
-            ".reg .b32 %rhs;\n\t"
-            ".reg .b32 %amt2;\n\t"
-            "shr.b32 \t%lhs, $src, $amt;\n\t"
-            "sub.s32 \t%amt2, 32, $amt;\n\t"
-            "shl.b32 \t%rhs, $src, %amt2;\n\t"
-            "add.u32 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            [(set Int32Regs:$dst, (rotr (i32 Int32Regs:$src), (i32 Int32Regs:$amt)))]>,
-           Requires<[noHWROT32]>;
-
-// 64-bit software rotate by immediate.  $amt2 should equal 64 - $amt1.
-def ROT64imm_sw :
-  NVPTXInst<(outs Int64Regs:$dst),
-            (ins Int64Regs:$src, i32imm:$amt1, i32imm:$amt2),
-            "{{\n\t"
-            ".reg .b64 %lhs;\n\t"
-            ".reg .b64 %rhs;\n\t"
-            "shl.b64 \t%lhs, $src, $amt1;\n\t"
-            "shr.b64 \t%rhs, $src, $amt2;\n\t"
-            "add.u64 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            []>;
-
-def SUB_FRM_64 : SDNodeXForm<imm, [{
-    return CurDAG->getTargetConstant(64-N->getZExtValue(), SDLoc(N), MVT::i32);
-}]>;
-
-def : Pat<(rotl Int64Regs:$src, (i32 imm:$amt)),
-          (ROT64imm_sw Int64Regs:$src, imm:$amt, (SUB_FRM_64 node:$amt))>;
-def : Pat<(rotr Int64Regs:$src, (i32 imm:$amt)),
-          (ROT64imm_sw Int64Regs:$src, (SUB_FRM_64 node:$amt), imm:$amt)>;
-
-// 64-bit software rotate left by register.
-def ROTL64reg_sw :
-  NVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$src, Int32Regs:$amt),
-            "{{\n\t"
-            ".reg .b64 %lhs;\n\t"
-            ".reg .b64 %rhs;\n\t"
-            ".reg .u32 %amt2;\n\t"
-            "and.b32 \t%amt2, $amt, 63;\n\t"
-            "shl.b64 \t%lhs, $src, %amt2;\n\t"
-            "sub.u32 \t%amt2, 64, %amt2;\n\t"
-            "shr.b64 \t%rhs, $src, %amt2;\n\t"
-            "add.u64 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            [(set Int64Regs:$dst, (rotl Int64Regs:$src, (i32 Int32Regs:$amt)))]>;
-
-def ROTR64reg_sw :
-  NVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$src, Int32Regs:$amt),
-            "{{\n\t"
-            ".reg .b64 %lhs;\n\t"
-            ".reg .b64 %rhs;\n\t"
-            ".reg .u32 %amt2;\n\t"
-            "and.b32 \t%amt2, $amt, 63;\n\t"
-            "shr.b64 \t%lhs, $src, %amt2;\n\t"
-            "sub.u32 \t%amt2, 64, %amt2;\n\t"
-            "shl.b64 \t%rhs, $src, %amt2;\n\t"
-            "add.u64 \t$dst, %lhs, %rhs;\n\t"
-            "}}",
-            [(set Int64Regs:$dst, (rotr Int64Regs:$src, (i32 Int32Regs:$amt)))]>;
-
-//
-// Funnnel shift in clamp mode
-//
-
-// Create SDNodes so they can be used in the DAG code, e.g.
-// NVPTXISelLowering (LowerShiftLeftParts and LowerShiftRightParts)
-def FUN_SHFL_CLAMP : SDNode<"NVPTXISD::FUN_SHFL_CLAMP", SDTIntShiftDOp, []>;
-def FUN_SHFR_CLAMP : SDNode<"NVPTXISD::FUN_SHFR_CLAMP", SDTIntShiftDOp, []>;
-
-def FUNSHFLCLAMP :
-  NVPTXInst<(outs Int32Regs:$dst),
-            (ins Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
-            "shf.l.clamp.b32 \t$dst, $lo, $hi, $amt;",
-            [(set Int32Regs:$dst,
-              (FUN_SHFL_CLAMP (i32 Int32Regs:$lo), (i32 Int32Regs:$hi), (i32 Int32Regs:$amt)))]>;
-
-def FUNSHFRCLAMP :
-  NVPTXInst<(outs Int32Regs:$dst),
-            (ins Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
-            "shf.r.clamp.b32 \t$dst, $lo, $hi, $amt;",
-            [(set Int32Regs:$dst,
-             (FUN_SHFR_CLAMP (i32 Int32Regs:$lo), (i32 Int32Regs:$hi), (i32 Int32Regs:$amt)))]>;
 
 //
 // BFE - bit-field extract
@@ -3657,6 +3496,42 @@ def : Pat<(v2i16 (build_vector (i16 Int16Regs:$a), (i16 Int16Regs:$b))),
 def: Pat<(v2i16 (scalar_to_vector (i16 Int16Regs:$a))),
          (CVT_u32_u16 Int16Regs:$a, CvtNONE)>;
 
+//
+// Funnel-Shift
+//
+
+// Create SDNodes so they can be used in the DAG code, e.g.
+// NVPTXISelLowering (LowerShiftLeftParts and LowerShiftRightParts)
+def fshl_clamp : SDNode<"NVPTXISD::FSHL_CLAMP", SDTIntShiftDOp, []>;
+def fshr_clamp : SDNode<"NVPTXISD::FSHR_CLAMP", SDTIntShiftDOp, []>;
+
+// Funnel shift, requires >= sm_32.  Does not trap if amt is out of range, so
+// no side effects.
+let hasSideEffects = false in {
+  multiclass ShfInst<string mode, SDNode op> {
+    def _i
+      : NVPTXInst<(outs Int32Regs:$dst),
+                  (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
+                  "shf." # mode # ".b32 \t$dst, $lo, $hi, $amt;",
+                  [(set Int32Regs:$dst,
+                      (op (i32 Int32Regs:$hi), (i32 Int32Regs:$lo), (i32 imm:$amt)))]>,
+        Requires<[hasHWROT32]>;
+
+    def _r
+      : NVPTXInst<(outs Int32Regs:$dst),
+                  (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
+                  "shf." # mode # ".b32 \t$dst, $lo, $hi, $amt;",
+                  [(set Int32Regs:$dst,
+                      (op (i32 Int32Regs:$hi), (i32 Int32Regs:$lo), (i32 Int32Regs:$amt)))]>,
+        Requires<[hasHWROT32]>;
+  }
+
+  defm SHF_L_CLAMP : ShfInst<"l.clamp", fshl_clamp>;
+  defm SHF_R_CLAMP : ShfInst<"r.clamp", fshr_clamp>;
+  defm SHF_L_WRAP  : ShfInst<"l.wrap", fshl>;
+  defm SHF_R_WRAP  : ShfInst<"r.wrap", fshr>;
+}
+
 // Count leading zeros
 let hasSideEffects = false in {
   def CLZr32 : NVPTXInst<(outs Int32Regs:$d), (ins Int32Regs:$a),
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index f22f0b368c9d5..176d28c991207 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2739,134 +2739,9 @@ def : Pat<(int_nvvm_read_ptx_sreg_envreg30), (MOV_SPECIAL ENVREG30)>;
 def : Pat<(int_nvvm_read_ptx_sreg_envreg31), (MOV_SPECIAL ENVREG31)>;
 
 
-// rotate builtin support
-
-def ROTATE_B32_HW_IMM
-  : NVPTXInst<(outs Int32Regs:$dst),
-              (ins  Int32Regs:$src, i32imm:$amt),
-              "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
-              [(set Int32Regs:$dst,
-                 (int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)))]>,
-              Requires<[hasHWROT32]> ;
-
-def ROTATE_B32_HW_REG
-  : NVPTXInst<(outs Int32Regs:$dst),
-              (ins  Int32Regs:$src, Int32Regs:$amt),
-              "shf.l.wrap.b32 \t$dst, $src, $src, $amt;",
-              [(set Int32Regs:$dst,
-                 (int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt))]>,
-              Requires<[hasHWROT32]> ;
-
-def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, (i32 imm:$amt)),
-          (ROT32imm_sw Int32Regs:$src, imm:$amt, (SUB_FRM_32 node:$amt))>,
-      Requires<[noHWROT32]> ;
-
-def : Pat<(int_nvvm_rotate_b32 Int32Regs:$src, Int32Regs:$amt),
-          (ROTL32reg_sw Int32Regs:$src, Int32Regs:$amt)>,
-      Requires<[noHWROT32]> ;
-
-let hasSideEffects = false in {
-  def GET_LO_INT64 : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
-    !strconcat("{{\n\t",
-               ".reg .b32 %dummy;\n\t",
-               "mov.b64 \t{$dst,%dummy}, $src;\n\t",
-               "}}"),
-          []> ;
-
-  def GET_HI_INT64 : NVPTXInst<(outs Int32Regs:$dst), (ins Int64Regs:$src),
-    !strconcat("{{\n\t",
-               ".reg .b32 %dummy;\n\t",
-               "mov.b64 \t{%dummy,$dst}, $src;\n\t",
-               "}}"),
-          []> ;
-}
-
-let hasSideEffects = false in {
-  def PACK_TWO_INT32
-    : NVPTXInst<(outs Int64Regs:$dst), (ins Int32Regs:$lo, Int32Regs:$hi),
-                "mov.b64 \t$dst, {{$lo, $hi}};", []> ;
-}
-
 def : Pat<(int_nvvm_swap_lo_hi_b64 Int64Regs:$src),
-          (PACK_TWO_INT32 (GET_HI_INT64 Int64Regs:$src),
-                          (GET_LO_INT64 Int64Regs:$src))> ;
-
-// Funnel shift, requires >= sm_32.  Does not trap if amt is out of range, so
-// no side effects.
-let hasSideEffects = false in {
-  def SHF_L_WRAP_B32_IMM
-    : NVPTXInst<(outs Int32Regs:$dst),
-                (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
-                "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
-      Requires<[hasHWROT32]>;
-
-  def SHF_L_WRAP_B32_REG
-    : NVPTXInst<(outs Int32Regs:$dst),
-                (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
-                "shf.l.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
-      Requires<[hasHWROT32]>;
-
-  def SHF_R_WRAP_B32_IMM
-    : NVPTXInst<(outs Int32Regs:$dst),
-                (ins  Int32Regs:$lo, Int32Regs:$hi, i32imm:$amt),
-                "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
-      Requires<[hasHWROT32]>;
-
-  def SHF_R_WRAP_B32_REG
-    : NVPTXInst<(outs Int32Regs:$dst),
-                (ins  Int32Regs:$lo, Int32Regs:$hi, Int32Regs:$amt),
-                "shf.r.wrap.b32 \t$dst, $lo, $hi, $amt;",[]>,
-      Requires<[hasHWROT32]>;
-}
-
-// HW version of rotate 64
-def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
-          (PACK_TWO_INT32
-            (SHF_L_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
-                                (GET_LO_INT64 Int64Regs:$src), imm:$amt),
-            (SHF_L_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
-                                (GET_HI_INT64 Int64Regs:$src), imm:$amt))>,
-      Requires<[hasHWROT32]>;
-
-def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
-          (PACK_TWO_INT32
-            (SHF_L_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
-                                (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt),
-            (SHF_L_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
-                               (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt))>,
-      Requires<[hasHWROT32]>;
-
-
-def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
-          (PACK_TWO_INT32
-            (SHF_R_WRAP_B32_IMM (GET_LO_INT64 Int64Regs:$src),
-                                (GET_HI_INT64 Int64Regs:$src), imm:$amt),
-            (SHF_R_WRAP_B32_IMM (GET_HI_INT64 Int64Regs:$src),
-                                (GET_LO_INT64 Int64Regs:$src), imm:$amt))>,
-      Requires<[hasHWROT32]>;
-
-def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
-          (PACK_TWO_INT32
-            (SHF_R_WRAP_B32_REG (GET_LO_INT64 Int64Regs:$src),
-                                (GET_HI_INT64 Int64Regs:$src), Int32Regs:$amt),
-            (SHF_R_WRAP_B32_REG (GET_HI_INT64 Int64Regs:$src),
-                               (GET_LO_INT64 Int64Regs:$src), Int32Regs:$amt))>,
-      Requires<[hasHWROT32]>;
-
-// SW version of rotate 64
-def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, (i32 imm:$amt)),
-          (ROT64imm_sw Int64Regs:$src, imm:$amt, (SUB_FRM_64 node:$amt))>,
-      Requires<[noHWROT32]>;
-def : Pat<(int_nvvm_rotate_b64 Int64Regs:$src, Int32Regs:$amt),
-          (ROTL64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
-      Requires<[noHWROT32]>;
-def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, (i32 imm:$amt)),
-          (ROT64imm_sw Int64Regs:$src, (SUB_FRM_64 node:$amt), imm:$amt)>,
-      Requires<[noHWROT32]>;
-def : Pat<(int_nvvm_rotate_right_b64 Int64Regs:$src, Int32Regs:$amt),
-          (ROTR64reg_sw Int64Regs:$src, Int32Regs:$amt)>,
-      Requires<[noHWROT32]>;
-
+          (V2I32toI64 (I64toI32H Int64Regs:$src),
+                      (I64toI32L Int64Regs:$src))> ;
 
 //-----------------------------------
 // Texture Intrinsics
diff --git a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
index a8a23f04c1249..c30372fed1832 100644
--- a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "NVPTXRegisterInfo.h"
+#include "MCTargetDesc/NVPTXInstPrinter.h"
 #include "NVPTX.h"
 #include "NVPTXSubtarget.h"
 #include "NVPTXTargetMachine.h"
@@ -141,3 +142,44 @@ NVPTXRegisterInfo::getFrameLocalRegister(const MachineFunction &MF) const {
       static_cast<const NVPTXTargetMachine &>(MF.getTarget());
   return TM.is64Bit() ? NVPTX::VRFrameLocal64 : NVPTX::VRFrameLocal32;
 }
+
+void NVPTXRegisterInfo::clearDebugRegisterMap() const {
+  debugRegisterMap.clear();
+}
+
+static uint64_t encodeRegisterForDwarf(std::string registerName) {
+  if (registerName.length() > 8) {
+    // The name is more than 8 characters long, and so won't fit into 64 bits.
+    return 0;
+  }
+
+  // Encode the name string into a DWARF register number using cuda-gdb's
+  // encoding.  See cuda_check_dwarf2_reg_ptx_virtual_register in cuda-tdep.c,
+  // https://github.com/NVIDIA/cuda-gdb/blob/e5cf3bddae520ffb326f95b4d98ce5c7474b828b/gdb/cuda/cuda-tdep.c#L353
+  // IE the bytes of the string are concatenated in reverse into a single
+  // number, which is stored in ULEB128, but in practice must be no more than 8
+  // bytes (excluding null terminator, which is not included).
+  uint64_t result = 0;
+  for (unsigned char c : registerName)
+    result = (result << 8) | c;
+  return result;
+}
+
+void NVPTXRegisterInfo::addToDebugRegisterMap(
+    uint64_t preEncodedVirtualRegister, std::string registerName) const {
+  uint64_t mapped = encodeRegisterForDwarf(registerName);
+  if (mapped == 0)
+    return;
+  debugRegisterMap.insert({preEncodedVirtualRegister, mapped});
+}
+
+int64_t NVPTXRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const {
+  if (Register::isPhysicalRegister(RegNum)) {
+    std::string name = NVPTXInstPrinter::getRegisterName(RegNum.id());
+    return encodeRegisterForDwarf(name);
+  }
+  uint64_t lookup = debugRegisterMap.lookup(RegNum.id());
+  if (lookup)
+    return lookup;
+  return -1;
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.h b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.h
index 7bce3bd18ae8f..d2f6d257d6b07 100644
--- a/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXRegisterInfo.h
@@ -26,6 +26,10 @@ class NVPTXRegisterInfo : public NVPTXGenRegisterInfo {
   // Hold Strings that can be free'd all together with NVPTXRegisterInfo
   BumpPtrAllocator StrAlloc;
   UniqueStringSaver StrPool;
+  // State for debug register mapping that can be mutated even through a const
+  // pointer so that we can get the proper dwarf register encoding during ASM
+  // emission.
+  mutable DenseMap<uint64_t, uint64_t> debugRegisterMap;
 
 public:
   NVPTXRegisterInfo();
@@ -56,6 +60,18 @@ class NVPTXRegisterInfo : public NVPTXGenRegisterInfo {
     return getStrPool().save(O.str()).data();
   }
 
+  // Manage the debugRegisterMap.  PTX virtual registers for DebugInfo are
+  // encoded using the names used in the emitted text of the PTX assembly. This
+  // mapping must be managed during assembly emission.
+  //
+  // These are marked const because the interfaces used to access this
+  // RegisterInfo object are all const, but we need to communicate some state
+  // here, because the proper encoding for debug registers is available only
+  // temporarily during ASM emission.
+  void addToDebugRegisterMap(uint64_t preEncodedVirtualRegister,
+                             std::string registerName) const;
+  void clearDebugRegisterMap() const;
+  int64_t getDwarfRegNum(MCRegister RegNum, bool isEH) const override;
 };
 
 std::string getNVPTXRegClassName(const TargetRegisterClass *RC);
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index be1c87d07f4de..2d62f34726e88 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -272,6 +272,18 @@ std::optional<unsigned> getMaxNTID(const Function &F) {
   return std::nullopt;
 }
 
+std::optional<unsigned> getClusterDimx(const Function &F) {
+  return findOneNVVMAnnotation(&F, "cluster_dim_x");
+}
+
+std::optional<unsigned> getClusterDimy(const Function &F) {
+  return findOneNVVMAnnotation(&F, "cluster_dim_y");
+}
+
+std::optional<unsigned> getClusterDimz(const Function &F) {
+  return findOneNVVMAnnotation(&F, "cluster_dim_z");
+}
+
 std::optional<unsigned> getMaxClusterRank(const Function &F) {
   return findOneNVVMAnnotation(&F, "maxclusterrank");
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index cf15dff85cbde..36fc0e4915353 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -55,6 +55,10 @@ std::optional<unsigned> getReqNTIDy(const Function &);
 std::optional<unsigned> getReqNTIDz(const Function &);
 std::optional<unsigned> getReqNTID(const Function &);
 
+std::optional<unsigned> getClusterDimx(const Function &);
+std::optional<unsigned> getClusterDimy(const Function &);
+std::optional<unsigned> getClusterDimz(const Function &);
+
 std::optional<unsigned> getMaxClusterRank(const Function &);
 std::optional<unsigned> getMinCTASm(const Function &);
 std::optional<unsigned> getMaxNReg(const Function &);
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 5e29a92f0bacd..fbad7d5d02db6 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -480,7 +480,13 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            RISCVMCRegisterClasses[RISCV::GPRRegClassID].contains(Reg.RegNum);
   }
 
+  bool isGPRF16() const {
+    return Kind == KindTy::Register &&
+           RISCVMCRegisterClasses[RISCV::GPRF16RegClassID].contains(Reg.RegNum);
+  }
+
   bool isGPRAsFPR() const { return isGPR() && Reg.IsGPRAsFPR; }
+  bool isGPRAsFPR16() const { return isGPRF16() && Reg.IsGPRAsFPR; }
   bool isGPRPairAsFPR() const { return isGPRPair() && Reg.IsGPRAsFPR; }
 
   bool isGPRPair() const {
@@ -1342,6 +1348,10 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     Op.Reg.RegNum = convertFPR64ToFPR16(Reg);
     return Match_Success;
   }
+  if (Kind == MCK_GPRAsFPR16 && Op.isGPRAsFPR()) {
+    Op.Reg.RegNum = Reg - RISCV::X0 + RISCV::X0_H;
+    return Match_Success;
+  }
 
   // There are some GPRF64AsFPR instructions that have no RV32 equivalent. We
   // reject them at parsing thinking we should match as GPRPairAsFPR for RV32.
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index b869458a25614..c2659a51b0209 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -81,6 +81,19 @@ static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, uint32_t RegNo,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeGPRF16RegisterClass(MCInst &Inst, uint32_t RegNo,
+                                              uint64_t Address,
+                                              const MCDisassembler *Decoder) {
+  bool IsRVE = Decoder->getSubtargetInfo().hasFeature(RISCV::FeatureStdExtE);
+
+  if (RegNo >= 32 || (IsRVE && RegNo >= 16))
+    return MCDisassembler::Fail;
+
+  MCRegister Reg = RISCV::X0_H + RegNo;
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus DecodeGPRX1X5RegisterClass(MCInst &Inst, uint32_t RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
index b7ed9de6ca84d..d610f0b956027 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
@@ -129,7 +129,7 @@ ArrayRef<MCPhysReg> RISCV::getArgGPRs(const RISCVABI::ABI ABI) {
   static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
                                        RISCV::X13, RISCV::X14, RISCV::X15,
                                        RISCV::X16, RISCV::X17};
-  // The GPRs used for passing arguments in the ILP32E/ILP64E ABI.
+  // The GPRs used for passing arguments in the ILP32E/LP64E ABI.
   static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
                                        RISCV::X13, RISCV::X14, RISCV::X15};
 
@@ -139,6 +139,23 @@ ArrayRef<MCPhysReg> RISCV::getArgGPRs(const RISCVABI::ABI ABI) {
   return ArrayRef(ArgIGPRs);
 }
 
+static ArrayRef<MCPhysReg> getArgGPR16s(const RISCVABI::ABI ABI) {
+  // The GPRs used for passing arguments in the ILP32* and LP64* ABIs, except
+  // the ILP32E ABI.
+  static const MCPhysReg ArgIGPRs[] = {RISCV::X10_H, RISCV::X11_H, RISCV::X12_H,
+                                       RISCV::X13_H, RISCV::X14_H, RISCV::X15_H,
+                                       RISCV::X16_H, RISCV::X17_H};
+  // The GPRs used for passing arguments in the ILP32E/LP64E ABI.
+  static const MCPhysReg ArgEGPRs[] = {RISCV::X10_H, RISCV::X11_H,
+                                       RISCV::X12_H, RISCV::X13_H,
+                                       RISCV::X14_H, RISCV::X15_H};
+
+  if (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E)
+    return ArrayRef(ArgEGPRs);
+
+  return ArrayRef(ArgIGPRs);
+}
+
 static ArrayRef<MCPhysReg> getFastCCArgGPRs(const RISCVABI::ABI ABI) {
   // The GPRs used for passing arguments in the FastCC, X5 and X6 might be used
   // for save-restore libcall, so we don't use them.
@@ -147,7 +164,7 @@ static ArrayRef<MCPhysReg> getFastCCArgGPRs(const RISCVABI::ABI ABI) {
       RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15,
       RISCV::X16, RISCV::X17, RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31};
 
-  // The GPRs used for passing arguments in the FastCC when using ILP32E/ILP64E.
+  // The GPRs used for passing arguments in the FastCC when using ILP32E/LP64E.
   static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
                                           RISCV::X13, RISCV::X14, RISCV::X15};
 
@@ -157,6 +174,26 @@ static ArrayRef<MCPhysReg> getFastCCArgGPRs(const RISCVABI::ABI ABI) {
   return ArrayRef(FastCCIGPRs);
 }
 
+static ArrayRef<MCPhysReg> getFastCCArgGPRF16s(const RISCVABI::ABI ABI) {
+  // The GPRs used for passing arguments in the FastCC, X5 and X6 might be used
+  // for save-restore libcall, so we don't use them.
+  // Don't use X7 for fastcc, since Zicfilp uses X7 as the label register.
+  static const MCPhysReg FastCCIGPRs[] = {
+      RISCV::X10_H, RISCV::X11_H, RISCV::X12_H, RISCV::X13_H,
+      RISCV::X14_H, RISCV::X15_H, RISCV::X16_H, RISCV::X17_H,
+      RISCV::X28_H, RISCV::X29_H, RISCV::X30_H, RISCV::X31_H};
+
+  // The GPRs used for passing arguments in the FastCC when using ILP32E/LP64E.
+  static const MCPhysReg FastCCEGPRs[] = {RISCV::X10_H, RISCV::X11_H,
+                                          RISCV::X12_H, RISCV::X13_H,
+                                          RISCV::X14_H, RISCV::X15_H};
+
+  if (ABI == RISCVABI::ABI_ILP32E || ABI == RISCVABI::ABI_LP64E)
+    return ArrayRef(FastCCEGPRs);
+
+  return ArrayRef(FastCCIGPRs);
+}
+
 // Pass a 2*XLEN argument that has been split into two XLEN values through
 // registers or the stack as necessary.
 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
@@ -320,6 +357,13 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
     }
   }
 
+  if ((ValVT == MVT::f16 && Subtarget.hasStdExtZhinxmin())) {
+    if (MCRegister Reg = State.AllocateReg(getArgGPR16s(ABI))) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
   ArrayRef<MCPhysReg> ArgGPRs = RISCV::getArgGPRs(ABI);
 
   // Zfinx/Zdinx use GPR without a bitcast when possible.
@@ -564,9 +608,16 @@ bool llvm::CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
 
   MVT XLenVT = Subtarget.getXLenVT();
 
+  // Check if there is an available GPRF16 before hitting the stack.
+  if ((LocVT == MVT::f16 && Subtarget.hasStdExtZhinxmin())) {
+    if (MCRegister Reg = State.AllocateReg(getFastCCArgGPRF16s(ABI))) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
   // Check if there is an available GPR before hitting the stack.
-  if ((LocVT == MVT::f16 && Subtarget.hasStdExtZhinxmin()) ||
-      (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
+  if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
       (LocVT == MVT::f64 && Subtarget.is64Bit() &&
        Subtarget.hasStdExtZdinx())) {
     if (MCRegister Reg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
index cce0ffe16e5fe..713c7a0661def 100644
--- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
+++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp
@@ -93,14 +93,19 @@ bool RISCVDeadRegisterDefinitions::runOnMachineFunction(MachineFunction &MF) {
           continue;
         LLVM_DEBUG(dbgs() << "    Dead def operand #" << I << " in:\n      ";
                    MI.print(dbgs()));
+        Register X0Reg;
         const TargetRegisterClass *RC = TII->getRegClass(Desc, I, TRI, MF);
-        if (!(RC && RC->contains(RISCV::X0))) {
+        if (RC && RC->contains(RISCV::X0)) {
+          X0Reg = RISCV::X0;
+        } else if (RC && RC->contains(RISCV::X0_H)) {
+          X0Reg = RISCV::X0_H;
+        } else {
           LLVM_DEBUG(dbgs() << "    Ignoring, register is not a GPR.\n");
           continue;
         }
         assert(LIS.hasInterval(Reg));
         LIS.removeInterval(Reg);
-        MO.setReg(RISCV::X0);
+        MO.setReg(X0Reg);
         LLVM_DEBUG(dbgs() << "    Replacing with zero register. New:\n      ";
                    MI.print(dbgs()));
         ++NumDeadDefsReplaced;
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 72f96965ae985..2501256ca6adf 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -48,6 +48,8 @@ class RISCVExpandPseudo : public MachineFunctionPass {
                   MachineBasicBlock::iterator &NextMBBI);
   bool expandVMSET_VMCLR(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI, unsigned Opcode);
+  bool expandMV_FPR16INX(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI);
   bool expandRV32ZdinxStore(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI);
   bool expandRV32ZdinxLoad(MachineBasicBlock &MBB,
@@ -104,6 +106,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   // expanded instructions for each pseudo is correct in the Size field of the
   // tablegen definition for the pseudo.
   switch (MBBI->getOpcode()) {
+  case RISCV::PseudoMV_FPR16INX:
+    return expandMV_FPR16INX(MBB, MBBI);
   case RISCV::PseudoRV32ZdinxSD:
     return expandRV32ZdinxStore(MBB, MBBI);
   case RISCV::PseudoRV32ZdinxLD:
@@ -266,6 +270,23 @@ bool RISCVExpandPseudo::expandVMSET_VMCLR(MachineBasicBlock &MBB,
   return true;
 }
 
+bool RISCVExpandPseudo::expandMV_FPR16INX(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator MBBI) {
+  DebugLoc DL = MBBI->getDebugLoc();
+  const TargetRegisterInfo *TRI = STI->getRegisterInfo();
+  Register DstReg = TRI->getMatchingSuperReg(
+      MBBI->getOperand(0).getReg(), RISCV::sub_16, &RISCV::GPRRegClass);
+  Register SrcReg = TRI->getMatchingSuperReg(
+      MBBI->getOperand(1).getReg(), RISCV::sub_16, &RISCV::GPRRegClass);
+
+  BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), DstReg)
+      .addReg(SrcReg, getKillRegState(MBBI->getOperand(1).isKill()))
+      .addImm(0);
+
+  MBBI->eraseFromParent(); // The pseudo instruction is gone now.
+  return true;
+}
+
 // This function expands the PseudoRV32ZdinxSD for storing a double-precision
 // floating-point value into memory by generating an equivalent instruction
 // sequence for RV32.
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 05ba18bf8ebd8..23479c2edf1d9 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -928,7 +928,10 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     }
 
     SDNode *Res;
-    if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
+    if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
+      Res =
+          CurDAG->getTargetExtractSubreg(RISCV::sub_16, DL, VT, Imm).getNode();
+    } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
       Res = CurDAG->getMachineNode(
           Opc, DL, VT, Imm,
           CurDAG->getTargetConstant(RISCVFPRndMode::RNE, DL, XLenVT));
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3b61cb5dfe090..bd796efd836c7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1082,8 +1082,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
                          VT, Custom);
       MVT EltVT = VT.getVectorElementType();
       if (isTypeLegal(EltVT))
-        setOperationAction({ISD::SPLAT_VECTOR, ISD::EXPERIMENTAL_VP_SPLAT}, VT,
-                           Custom);
+        setOperationAction({ISD::SPLAT_VECTOR, ISD::EXPERIMENTAL_VP_SPLAT,
+                            ISD::EXTRACT_VECTOR_ELT},
+                           VT, Custom);
       else
         setOperationAction({ISD::SPLAT_VECTOR, ISD::EXPERIMENTAL_VP_SPLAT},
                            EltVT, Custom);
@@ -8990,6 +8991,16 @@ SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
   }
 
+  if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
+      EltVT == MVT::bf16) {
+    // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
+    MVT IntVT = VecVT.changeTypeToInteger();
+    SDValue IntVec = DAG.getBitcast(IntVT, Vec);
+    SDValue IntExtract =
+        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, IntVec, Idx);
+    return DAG.getNode(RISCVISD::FMV_H_X, DL, EltVT, IntExtract);
+  }
+
   // If this is a fixed vector, we need to convert it to a scalable vector.
   MVT ContainerVT = VecVT;
   if (VecVT.isFixedLengthVector()) {
@@ -19741,11 +19752,14 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
   // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
   // split it and then direct call can be matched by PseudoCALL.
+  bool CalleeIsLargeExternalSymbol = false;
   if (getTargetMachine().getCodeModel() == CodeModel::Large) {
     if (auto *S = dyn_cast<GlobalAddressSDNode>(Callee))
       Callee = getLargeGlobalAddress(S, DL, PtrVT, DAG);
-    else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee))
+    else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
       Callee = getLargeExternalSymbol(S, DL, PtrVT, DAG);
+      CalleeIsLargeExternalSymbol = true;
+    }
   } else if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
     const GlobalValue *GV = S->getGlobal();
     Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, RISCVII::MO_CALL);
@@ -19781,16 +19795,28 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   // Emit the call.
   SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
 
+  // Use software guarded branch for large code model non-indirect calls
+  // Tail call to external symbol will have a null CLI.CB and we need another
+  // way to determine the callsite type
+  bool NeedSWGuarded = false;
+  if (getTargetMachine().getCodeModel() == CodeModel::Large &&
+      Subtarget.hasStdExtZicfilp() &&
+      ((CLI.CB && !CLI.CB->isIndirectCall()) || CalleeIsLargeExternalSymbol))
+    NeedSWGuarded = true;
+
   if (IsTailCall) {
     MF.getFrameInfo().setHasTailCall();
-    SDValue Ret = DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
+    unsigned CallOpc =
+        NeedSWGuarded ? RISCVISD::SW_GUARDED_TAIL : RISCVISD::TAIL;
+    SDValue Ret = DAG.getNode(CallOpc, DL, NodeTys, Ops);
     if (CLI.CFIType)
       Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
     DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
     return Ret;
   }
 
-  Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
+  unsigned CallOpc = NeedSWGuarded ? RISCVISD::SW_GUARDED_CALL : RISCVISD::CALL;
+  Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
   if (CLI.CFIType)
     Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
@@ -20238,6 +20264,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(CZERO_EQZ)
   NODE_NAME_CASE(CZERO_NEZ)
   NODE_NAME_CASE(SW_GUARDED_BRIND)
+  NODE_NAME_CASE(SW_GUARDED_CALL)
+  NODE_NAME_CASE(SW_GUARDED_TAIL)
   NODE_NAME_CASE(TUPLE_INSERT)
   NODE_NAME_CASE(TUPLE_EXTRACT)
   NODE_NAME_CASE(SF_VC_XV_SE)
@@ -21462,9 +21490,8 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
   if (DI->getIntrinsicID() != Intrinsic::vector_deinterleave2)
     return false;
 
-  unsigned Factor = 2;
+  const unsigned Factor = 2;
 
-  VectorType *VTy = cast<VectorType>(DI->getOperand(0)->getType());
   VectorType *ResVTy = cast<VectorType>(DI->getType()->getContainedType(0));
   const DataLayout &DL = LI->getDataLayout();
 
@@ -21472,18 +21499,15 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
                                     LI->getPointerAddressSpace(), DL))
     return false;
 
-  Function *VlsegNFunc;
-  Value *VL, *Return;
+  Value *Return;
   Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
-  SmallVector<Value *, 10> Ops;
 
-  if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
-    VlsegNFunc = Intrinsic::getDeclaration(
+  if (auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
+    Function *VlsegNFunc = Intrinsic::getDeclaration(
         LI->getModule(), FixedVlsegIntrIds[Factor - 2],
         {ResVTy, LI->getPointerOperandType(), XLenTy});
-    VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
-    Ops.append({LI->getPointerOperand(), VL});
-    Return = Builder.CreateCall(VlsegNFunc, Ops);
+    Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
+    Return = Builder.CreateCall(VlsegNFunc, {LI->getPointerOperand(), VL});
   } else {
     static const Intrinsic::ID IntrIds[] = {
         Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
@@ -21497,15 +21521,15 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
         LI->getContext(), "riscv.vector.tuple",
         ScalableVectorType::get(Type::getInt8Ty(LI->getContext()),
                                 NumElts * SEW / 8),
-        2);
+        Factor);
 
-    VlsegNFunc = Intrinsic::getDeclaration(LI->getModule(), IntrIds[Factor - 2],
-                                           {VecTupTy, XLenTy});
-    VL = Constant::getAllOnesValue(XLenTy);
+    Function *VlsegNFunc = Intrinsic::getDeclaration(
+        LI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
+    Value *VL = Constant::getAllOnesValue(XLenTy);
 
-    Ops.append({PoisonValue::get(VecTupTy), LI->getPointerOperand(), VL,
-                ConstantInt::get(XLenTy, Log2_64(SEW))});
-    Value *Vlseg = Builder.CreateCall(VlsegNFunc, Ops);
+    Value *Vlseg = Builder.CreateCall(
+        VlsegNFunc, {PoisonValue::get(VecTupTy), LI->getPointerOperand(), VL,
+                     ConstantInt::get(XLenTy, Log2_64(SEW))});
 
     SmallVector<Type *, 2> AggrTypes{Factor, ResVTy};
     Return = PoisonValue::get(StructType::get(LI->getContext(), AggrTypes));
@@ -21533,26 +21557,23 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
   if (II->getIntrinsicID() != Intrinsic::vector_interleave2)
     return false;
 
-  unsigned Factor = 2;
+  const unsigned Factor = 2;
 
-  VectorType *VTy = cast<VectorType>(II->getType());
-  VectorType *InVTy = cast<VectorType>(II->getOperand(0)->getType());
+  VectorType *InVTy = cast<VectorType>(II->getArgOperand(0)->getType());
   const DataLayout &DL = SI->getDataLayout();
 
   if (!isLegalInterleavedAccessType(InVTy, Factor, SI->getAlign(),
                                     SI->getPointerAddressSpace(), DL))
     return false;
 
-  Function *VssegNFunc;
-  Value *VL;
   Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
 
-  if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
-    VssegNFunc = Intrinsic::getDeclaration(
+  if (auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
+    Function *VssegNFunc = Intrinsic::getDeclaration(
         SI->getModule(), FixedVssegIntrIds[Factor - 2],
         {InVTy, SI->getPointerOperandType(), XLenTy});
-    VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
-    Builder.CreateCall(VssegNFunc, {II->getOperand(0), II->getOperand(1),
+    Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
+    Builder.CreateCall(VssegNFunc, {II->getArgOperand(0), II->getArgOperand(1),
                                     SI->getPointerOperand(), VL});
   } else {
     static const Intrinsic::ID IntrIds[] = {
@@ -21567,19 +21588,20 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
         SI->getContext(), "riscv.vector.tuple",
         ScalableVectorType::get(Type::getInt8Ty(SI->getContext()),
                                 NumElts * SEW / 8),
-        2);
+        Factor);
 
-    VssegNFunc = Intrinsic::getDeclaration(SI->getModule(), IntrIds[Factor - 2],
-                                           {VecTupTy, XLenTy});
+    Function *VssegNFunc = Intrinsic::getDeclaration(
+        SI->getModule(), IntrIds[Factor - 2], {VecTupTy, XLenTy});
 
-    VL = Constant::getAllOnesValue(XLenTy);
+    Value *VL = Constant::getAllOnesValue(XLenTy);
 
     Function *VecInsertFunc = Intrinsic::getDeclaration(
         SI->getModule(), Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy});
     Value *StoredVal = PoisonValue::get(VecTupTy);
     for (unsigned i = 0; i < Factor; ++i)
-      StoredVal = Builder.CreateCall(
-          VecInsertFunc, {StoredVal, II->getOperand(i), Builder.getInt32(i)});
+      StoredVal =
+          Builder.CreateCall(VecInsertFunc, {StoredVal, II->getArgOperand(i),
+                                             Builder.getInt32(i)});
 
     Builder.CreateCall(VssegNFunc, {StoredVal, SI->getPointerOperand(), VL,
                                     ConstantInt::get(XLenTy, Log2_64(SEW))});
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index ceb9d49900284..05581552ab604 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -411,9 +411,12 @@ enum NodeType : unsigned {
   CZERO_EQZ, // vt.maskc for XVentanaCondOps.
   CZERO_NEZ, // vt.maskcn for XVentanaCondOps.
 
-  /// Software guarded BRIND node. Operand 0 is the chain operand and
-  /// operand 1 is the target address.
+  // Software guarded BRIND node. Operand 0 is the chain operand and
+  // operand 1 is the target address.
   SW_GUARDED_BRIND,
+  // Software guarded calls for large code model
+  SW_GUARDED_CALL,
+  SW_GUARDED_TAIL,
 
   SF_VC_XV_SE,
   SF_VC_IV_SE,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 8dafd824963c0..f0295d289ed86 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -104,6 +104,7 @@ Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
     MemBytes = 1;
     break;
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::FLH:
     MemBytes = 2;
@@ -144,6 +145,7 @@ Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
     MemBytes = 1;
     break;
   case RISCV::SH:
+  case RISCV::SH_INX:
   case RISCV::FSH:
     MemBytes = 2;
     break;
@@ -462,6 +464,13 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
+  if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
+    BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)
+        .addReg(SrcReg,
+                getKillRegState(KillSrc) | getRenamableRegState(RenamableSrc));
+    return;
+  }
+
   if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
     // Emit an ADDI for both parts of GPRPair.
     BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
@@ -583,6 +592,9 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::SW : RISCV::SD;
     IsScalableVector = false;
+  } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::SH_INX;
+    IsScalableVector = false;
   } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoRV32ZdinxSD;
     IsScalableVector = false;
@@ -666,6 +678,9 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::LW : RISCV::LD;
     IsScalableVector = false;
+  } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
+    Opcode = RISCV::LH_INX;
+    IsScalableVector = false;
   } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
     Opcode = RISCV::PseudoRV32ZdinxLD;
     IsScalableVector = false;
@@ -784,6 +799,24 @@ MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
       }
       break;
     }
+    if (RISCV::getRVVMCOpcode(MI.getOpcode()) == RISCV::VFMV_F_S) {
+      unsigned Log2SEW =
+          MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+      switch (Log2SEW) {
+      case 4:
+        // TODO: Support f16/bf16
+        return nullptr;
+      case 5:
+        LoadOpc = RISCV::FLW;
+        break;
+      case 6:
+        LoadOpc = RISCV::FLD;
+        break;
+      default:
+        llvm_unreachable("Unexpected SEW");
+      }
+      break;
+    }
     return nullptr;
   case RISCV::SEXT_H:
     LoadOpc = RISCV::LH;
@@ -1520,6 +1553,9 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   }
 
   switch (Opcode) {
+  case RISCV::PseudoMV_FPR16INX:
+    // MV is always compressible to either c.mv or c.li rd, 0.
+    return STI.hasStdExtCOrZca() ? 2 : 4;
   case TargetOpcode::STACKMAP:
     // The upper bound for a stackmap intrinsic is the full length of its shadow
     return StackMapOpers(&MI).getNumPatchBytes();
@@ -2575,6 +2611,7 @@ bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
   case RISCV::LB:
   case RISCV::LBU:
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::LW:
   case RISCV::LWU:
@@ -2584,6 +2621,7 @@ bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
   case RISCV::FLD:
   case RISCV::SB:
   case RISCV::SH:
+  case RISCV::SH_INX:
   case RISCV::SW:
   case RISCV::SD:
   case RISCV::FSH:
@@ -2647,9 +2685,11 @@ bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
   case RISCV::LBU:
   case RISCV::SB:
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::FLH:
   case RISCV::SH:
+  case RISCV::SH_INX:
   case RISCV::FSH:
   case RISCV::LW:
   case RISCV::LWU:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index fe5623e2920e2..a2f1e3ded18fe 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -57,6 +57,9 @@ def callseq_end   : SDNode<"ISD::CALLSEQ_END", SDT_CallSeqEnd,
 def riscv_call      : SDNode<"RISCVISD::CALL", SDT_RISCVCall,
                              [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                               SDNPVariadic]>;
+def riscv_sw_guarded_call : SDNode<"RISCVISD::SW_GUARDED_CALL", SDT_RISCVCall,
+                                   [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+                                    SDNPVariadic]>;
 def riscv_ret_glue  : SDNode<"RISCVISD::RET_GLUE", SDTNone,
                              [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
 def riscv_sret_glue : SDNode<"RISCVISD::SRET_GLUE", SDTNone,
@@ -69,6 +72,9 @@ def riscv_brcc      : SDNode<"RISCVISD::BR_CC", SDT_RISCVBrCC,
 def riscv_tail      : SDNode<"RISCVISD::TAIL", SDT_RISCVCall,
                              [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                               SDNPVariadic]>;
+def riscv_sw_guarded_tail : SDNode<"RISCVISD::SW_GUARDED_TAIL", SDT_RISCVCall,
+                                   [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+                                    SDNPVariadic]>;
 def riscv_sw_guarded_brind : SDNode<"RISCVISD::SW_GUARDED_BRIND",
                                     SDTBrind, [SDNPHasChain]>;
 def riscv_sllw      : SDNode<"RISCVISD::SLLW", SDT_RISCVIntBinOpW>;
@@ -514,8 +520,8 @@ class BranchCC_rri<bits<3> funct3, string opcodestr>
 }
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in {
-class Load_ri<bits<3> funct3, string opcodestr>
-    : RVInstI<funct3, OPC_LOAD, (outs GPR:$rd), (ins GPRMem:$rs1, simm12:$imm12),
+class Load_ri<bits<3> funct3, string opcodestr, DAGOperand rty = GPR>
+    : RVInstI<funct3, OPC_LOAD, (outs rty:$rd), (ins GPRMem:$rs1, simm12:$imm12),
               opcodestr, "$rd, ${imm12}(${rs1})">;
 
 class HLoad_r<bits<7> funct7, bits<5> funct5, string opcodestr>
@@ -529,9 +535,9 @@ class HLoad_r<bits<7> funct7, bits<5> funct5, string opcodestr>
 // reflecting the order these fields are specified in the instruction
 // encoding.
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in {
-class Store_rri<bits<3> funct3, string opcodestr>
+class Store_rri<bits<3> funct3, string opcodestr, DAGOperand rty = GPR>
     : RVInstS<funct3, OPC_STORE, (outs),
-              (ins GPR:$rs2, GPRMem:$rs1, simm12:$imm12),
+              (ins rty:$rs2, GPRMem:$rs1, simm12:$imm12),
               opcodestr, "$rs2, ${imm12}(${rs1})">;
 
 class HStore_rr<bits<7> funct7, string opcodestr>
@@ -543,8 +549,8 @@ class HStore_rr<bits<7> funct7, string opcodestr>
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class ALU_ri<bits<3> funct3, string opcodestr>
-    : RVInstI<funct3, OPC_OP_IMM, (outs GPR:$rd), (ins GPR:$rs1, simm12:$imm12),
+class ALU_ri<bits<3> funct3, string opcodestr, DAGOperand rty = GPR>
+    : RVInstI<funct3, OPC_OP_IMM, (outs rty:$rd), (ins rty:$rs1, simm12:$imm12),
               opcodestr, "$rd, $rs1, $imm12">,
       Sched<[WriteIALU, ReadIALU]>;
 
@@ -1555,10 +1561,15 @@ let Predicates = [NoStdExtZicfilp] in
 def PseudoCALLIndirect : Pseudo<(outs), (ins GPRJALR:$rs1),
                                 [(riscv_call GPRJALR:$rs1)]>,
                          PseudoInstExpansion<(JALR X1, GPR:$rs1, 0)>;
-let Predicates = [HasStdExtZicfilp] in
+let Predicates = [HasStdExtZicfilp] in {
 def PseudoCALLIndirectNonX7 : Pseudo<(outs), (ins GPRJALRNonX7:$rs1),
                                      [(riscv_call GPRJALRNonX7:$rs1)]>,
                               PseudoInstExpansion<(JALR X1, GPR:$rs1, 0)>;
+// For large code model, non-indirect calls could be software-guarded
+def PseudoCALLIndirectX7 : Pseudo<(outs), (ins GPRX7:$rs1),
+                                  [(riscv_sw_guarded_call GPRX7:$rs1)]>,
+                           PseudoInstExpansion<(JALR X1, GPR:$rs1, 0)>;
+}
 }
 
 let isBarrier = 1, isReturn = 1, isTerminator = 1 in
@@ -1579,10 +1590,15 @@ let Predicates = [NoStdExtZicfilp] in
 def PseudoTAILIndirect : Pseudo<(outs), (ins GPRTC:$rs1),
                                 [(riscv_tail GPRTC:$rs1)]>,
                          PseudoInstExpansion<(JALR X0, GPR:$rs1, 0)>;
-let Predicates = [HasStdExtZicfilp] in
+let Predicates = [HasStdExtZicfilp] in {
 def PseudoTAILIndirectNonX7 : Pseudo<(outs), (ins GPRTCNonX7:$rs1),
                                      [(riscv_tail GPRTCNonX7:$rs1)]>,
                               PseudoInstExpansion<(JALR X0, GPR:$rs1, 0)>;
+// For large code model, non-indirect calls could be software-guarded
+def PseudoTAILIndirectX7 : Pseudo<(outs), (ins GPRX7:$rs1),
+                                  [(riscv_sw_guarded_tail GPRX7:$rs1)]>,
+                           PseudoInstExpansion<(JALR X0, GPR:$rs1, 0)>;
+}
 }
 
 def : Pat<(riscv_tail (iPTR tglobaladdr:$dst)),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
index 11c2695a59854..bff740a33c1c1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZc.td
@@ -112,8 +112,9 @@ class CLoadB_ri<bits<6> funct6, string OpcodeStr>
 }
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
-class CLoadH_ri<bits<6> funct6, bit funct1, string OpcodeStr>
-    : RVInst16CLH<funct6, funct1, 0b00, (outs GPRC:$rd),
+class CLoadH_ri<bits<6> funct6, bit funct1, string OpcodeStr,
+                DAGOperand rty = GPRC>
+    : RVInst16CLH<funct6, funct1, 0b00, (outs rty:$rd),
                   (ins GPRCMem:$rs1, uimm2_lsb0:$imm),
                   OpcodeStr, "$rd, ${imm}(${rs1})"> {
   bits<2> imm;
@@ -132,9 +133,10 @@ class CStoreB_rri<bits<6> funct6, string OpcodeStr>
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
-class CStoreH_rri<bits<6> funct6, bit funct1, string OpcodeStr>
+class CStoreH_rri<bits<6> funct6, bit funct1, string OpcodeStr,
+                  DAGOperand rty = GPRC>
     : RVInst16CSH<funct6, funct1, 0b00, (outs),
-                  (ins GPRC:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm),
+                  (ins rty:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm),
                   OpcodeStr, "$rs2, ${imm}(${rs1})"> {
   bits<2> imm;
 
@@ -202,7 +204,15 @@ def C_SB : CStoreB_rri<0b100010, "c.sb">,
            Sched<[WriteSTB, ReadStoreData, ReadMemBase]>;
 def C_SH : CStoreH_rri<0b100011, 0b0, "c.sh">,
            Sched<[WriteSTH, ReadStoreData, ReadMemBase]>;
+
+// Compressed versions of Zhinx load/store.
+let isCodeGenOnly = 1 in {
+def C_LH_INX : CLoadH_ri<0b100001, 0b1, "c.lh", GPRF16C>,
+               Sched<[WriteLDH, ReadMemBase]>;
+def C_SH_INX : CStoreH_rri<0b100011, 0b0, "c.sh", GPRF16C>,
+               Sched<[WriteSTH, ReadStoreData, ReadMemBase]>;
 }
+} // Predicates = [HasStdExtZcb]
 
 // Zcmp
 let DecoderNamespace = "RVZcmp", Predicates = [HasStdExtZcmp],
@@ -318,6 +328,13 @@ def : CompressPat<(SB GPRC:$rs2, GPRCMem:$rs1, uimm2:$imm),
                   (C_SB GPRC:$rs2, GPRCMem:$rs1, uimm2:$imm)>;
 def : CompressPat<(SH GPRC:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm),
                   (C_SH GPRC:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm)>;
+
+let isCompressOnly = true in {
+def : CompressPat<(LH_INX GPRF16C:$rd, GPRCMem:$rs1, uimm2_lsb0:$imm),
+                  (C_LH_INX GPRF16C:$rd, GPRCMem:$rs1, uimm2_lsb0:$imm)>;
+def : CompressPat<(SH_INX GPRF16C:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm),
+                  (C_SH_INX GPRF16C:$rs2, GPRCMem:$rs1, uimm2_lsb0:$imm)>;
+}
 }// Predicates = [HasStdExtZcb]
 
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
index 792cb7fa6dbc2..51123180d47c6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -33,9 +33,14 @@ def riscv_fmv_x_signexth
 
 // Zhinxmin and Zhinx
 
+def GPRAsFPR16 : AsmOperandClass {
+  let Name = "GPRAsFPR16";
+  let ParserMethod = "parseGPRAsFPR";
+  let RenderMethod = "addRegOperands";
+}
+
 def FPR16INX : RegisterOperand<GPRF16> {
-  let ParserMatchClass = GPRAsFPR;
-  let DecoderMethod = "DecodeGPRRegisterClass";
+  let ParserMatchClass = GPRAsFPR16;
 }
 
 def ZfhExt     : ExtInfo<"", "", [HasStdExtZfh],
@@ -84,6 +89,19 @@ def FLH : FPLoad_r<0b001, "flh", FPR16, WriteFLD16>;
 def FSH : FPStore_r<0b001, "fsh", FPR16, WriteFST16>;
 } // Predicates = [HasHalfFPLoadStoreMove]
 
+let Predicates = [HasStdExtZhinxmin], isCodeGenOnly = 1 in {
+def LH_INX : Load_ri<0b001, "lh", GPRF16>, Sched<[WriteLDH, ReadMemBase]>;
+def SH_INX : Store_rri<0b001, "sh", GPRF16>,
+             Sched<[WriteSTH, ReadStoreData, ReadMemBase]>;
+
+// ADDI with GPRF16 register class to use for copy. This should not be used as
+// general ADDI, so the immediate should always be zero.
+let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveReg = 1,
+    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+def PseudoMV_FPR16INX : Pseudo<(outs GPRF16:$rd), (ins GPRF16:$rs), []>,
+                        Sched<[WriteIALU, ReadIALU]>;
+}
+
 foreach Ext = ZfhExts in {
   let SchedRW = [WriteFMA16, ReadFMA16, ReadFMA16, ReadFMA16Addend] in {
     defm FMADD_H  : FPFMA_rrr_frm_m<OPC_MADD,  0b10, "fmadd.h",  Ext>;
@@ -426,13 +444,10 @@ let Predicates = [HasStdExtZhinxmin] in {
 defm Select_FPR16INX : SelectCC_GPR_rrirr<FPR16INX, f16>;
 
 /// Loads
-def : Pat<(f16 (load (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12))),
-          (COPY_TO_REGCLASS (LH GPR:$rs1, simm12:$imm12), GPRF16)>;
+def : LdPat<load, LH_INX, f16>;
 
 /// Stores
-def : Pat<(store (f16 FPR16INX:$rs2),
-                 (AddrRegImm (XLenVT GPR:$rs1), simm12:$imm12)),
-          (SH (COPY_TO_REGCLASS FPR16INX:$rs2, GPR), GPR:$rs1, simm12:$imm12)>;
+def : StPat<store, SH_INX, GPRF16, f16>;
 } // Predicates = [HasStdExtZhinxmin]
 
 let Predicates = [HasStdExtZfhmin] in {
@@ -458,8 +473,8 @@ def : Pat<(any_fpround FPR32INX:$rs1), (FCVT_H_S_INX FPR32INX:$rs1, FRM_DYN)>;
 def : Pat<(any_fpextend FPR16INX:$rs1), (FCVT_S_H_INX FPR16INX:$rs1, FRM_RNE)>;
 
 // Moves (no conversion)
-def : Pat<(f16 (riscv_fmv_h_x GPR:$src)), (COPY_TO_REGCLASS GPR:$src, GPR)>;
-def : Pat<(riscv_fmv_x_anyexth FPR16INX:$src), (COPY_TO_REGCLASS FPR16INX:$src, GPR)>;
+def : Pat<(f16 (riscv_fmv_h_x GPR:$src)), (EXTRACT_SUBREG GPR:$src, sub_16)>;
+def : Pat<(riscv_fmv_x_anyexth FPR16INX:$src), (INSERT_SUBREG (XLenVT (IMPLICIT_DEF)), FPR16INX:$src, sub_16)>;
 
 def : Pat<(fcopysign FPR32INX:$rs1, FPR16INX:$rs2), (FSGNJ_S_INX $rs1, (FCVT_S_H_INX $rs2, FRM_RNE))>;
 } // Predicates = [HasStdExtZhinxmin]
diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
index 3f423450618df..5973e5bf2e525 100644
--- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp
@@ -103,8 +103,10 @@ static unsigned log2LdstWidth(unsigned Opcode) {
   case RISCV::SB:
     return 0;
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::SH:
+  case RISCV::SH_INX:
     return 1;
   case RISCV::LW:
   case RISCV::SW:
@@ -128,8 +130,10 @@ static unsigned offsetMask(unsigned Opcode) {
   case RISCV::SB:
     return maskTrailingOnes<unsigned>(2U);
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
   case RISCV::SH:
+  case RISCV::SH_INX:
     return maskTrailingOnes<unsigned>(1U);
   case RISCV::LW:
   case RISCV::SW:
@@ -173,6 +177,7 @@ static int64_t getBaseAdjustForCompression(int64_t Offset, unsigned Opcode) {
 // Return true if Reg is in a compressed register class.
 static bool isCompressedReg(Register Reg) {
   return RISCV::GPRCRegClass.contains(Reg) ||
+         RISCV::GPRF16CRegClass.contains(Reg) ||
          RISCV::FPR32CRegClass.contains(Reg) ||
          RISCV::FPR64CRegClass.contains(Reg);
 }
@@ -186,6 +191,7 @@ static bool isCompressibleLoad(const MachineInstr &MI) {
     return false;
   case RISCV::LBU:
   case RISCV::LH:
+  case RISCV::LH_INX:
   case RISCV::LHU:
     return STI.hasStdExtZcb();
   case RISCV::LW:
@@ -207,6 +213,7 @@ static bool isCompressibleStore(const MachineInstr &MI) {
     return false;
   case RISCV::SB:
   case RISCV::SH:
+  case RISCV::SH_INX:
     return STI.hasStdExtZcb();
   case RISCV::SW:
   case RISCV::SD:
@@ -320,6 +327,8 @@ static Register analyzeCompressibleUses(MachineInstr &FirstMI,
   // Work out the compressed register class from which to scavenge.
   if (RISCV::GPRRegClass.contains(RegImm.Reg))
     RCToScavenge = &RISCV::GPRCRegClass;
+  else if (RISCV::GPRF16RegClass.contains(RegImm.Reg))
+    RCToScavenge = &RISCV::GPRF16CRegClass;
   else if (RISCV::FPR32RegClass.contains(RegImm.Reg))
     RCToScavenge = &RISCV::FPR32CRegClass;
   else if (RISCV::FPR64RegClass.contains(RegImm.Reg))
@@ -410,6 +419,11 @@ bool RISCVMakeCompressibleOpt::runOnMachineFunction(MachineFunction &Fn) {
         BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(RISCV::ADDI), NewReg)
             .addReg(RegImm.Reg)
             .addImm(RegImm.Imm);
+      } else if (RISCV::GPRF16RegClass.contains(RegImm.Reg)) {
+        assert(RegImm.Imm == 0);
+        BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(RISCV::PseudoMV_FPR16INX),
+                NewReg)
+            .addReg(RegImm.Reg);
       } else {
         // If we are looking at replacing an FPR register we don't expect to
         // have any offset. The only compressible FP instructions with an offset
diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
index b6ac3384e7d3e..b3a2877edde4e 100644
--- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
@@ -385,6 +385,7 @@ bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
       return false;
     case RISCV::LB:
     case RISCV::LH:
+    case RISCV::LH_INX:
     case RISCV::LW:
     case RISCV::LBU:
     case RISCV::LHU:
@@ -395,6 +396,7 @@ bool RISCVMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi,
     case RISCV::FLD:
     case RISCV::SB:
     case RISCV::SH:
+    case RISCV::SH_INX:
     case RISCV::SW:
     case RISCV::SD:
     case RISCV::FSH:
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 91d539a355ac2..a8b6be4fe277a 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -115,11 +115,11 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   }
 
   // Use markSuperRegs to ensure any register aliases are also reserved
-  markSuperRegs(Reserved, RISCV::X2); // sp
-  markSuperRegs(Reserved, RISCV::X3); // gp
-  markSuperRegs(Reserved, RISCV::X4); // tp
+  markSuperRegs(Reserved, RISCV::X2_H); // sp
+  markSuperRegs(Reserved, RISCV::X3_H); // gp
+  markSuperRegs(Reserved, RISCV::X4_H); // tp
   if (TFI->hasFP(MF))
-    markSuperRegs(Reserved, RISCV::X8); // fp
+    markSuperRegs(Reserved, RISCV::X8_H); // fp
   // Reserve the base register if we need to realign the stack and allocate
   // variable-sized objects at runtime.
   if (TFI->hasBP(MF))
@@ -131,7 +131,7 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
   // There are only 16 GPRs for RVE.
   if (Subtarget.hasStdExtE())
-    for (MCPhysReg Reg = RISCV::X16; Reg <= RISCV::X31; Reg++)
+    for (MCPhysReg Reg = RISCV::X16_H; Reg <= RISCV::X31_H; Reg++)
       markSuperRegs(Reserved, Reg);
 
   // V registers for code generation. We handle them manually.
@@ -150,8 +150,8 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
     if (Subtarget.hasStdExtE())
       report_fatal_error("Graal reserved registers do not exist in RVE");
-    markSuperRegs(Reserved, RISCV::X23);
-    markSuperRegs(Reserved, RISCV::X27);
+    markSuperRegs(Reserved, RISCV::X23_H);
+    markSuperRegs(Reserved, RISCV::X27_H);
   }
 
   // Shadow stack pointer.
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 5725d8eda88ce..9cb589f2441a2 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -82,42 +82,84 @@ def sub_gpr_odd  : SubRegIndex<32, 32> {
 // instructions.
 
 let RegAltNameIndices = [ABIRegAltName] in {
+  // 16-bit sub-registers for use by Zhinx. Having a 16-bit sub-register reduces
+  // the spill size for these operations.
   let isConstant = true in
-  def X0  : RISCVReg<0, "x0", ["zero"]>, DwarfRegNum<[0]>;
+  def X0_H  : RISCVReg<0, "x0", ["zero"]>;
   let CostPerUse = [0, 1] in {
-  def X1  : RISCVReg<1, "x1", ["ra"]>, DwarfRegNum<[1]>;
-  def X2  : RISCVReg<2, "x2", ["sp"]>, DwarfRegNum<[2]>;
-  def X3  : RISCVReg<3, "x3", ["gp"]>, DwarfRegNum<[3]>;
-  def X4  : RISCVReg<4, "x4", ["tp"]>, DwarfRegNum<[4]>;
-  def X5  : RISCVReg<5, "x5", ["t0"]>, DwarfRegNum<[5]>;
-  def X6  : RISCVReg<6, "x6", ["t1"]>, DwarfRegNum<[6]>;
-  def X7  : RISCVReg<7, "x7", ["t2"]>, DwarfRegNum<[7]>;
+  def X1_H  : RISCVReg<1, "x1", ["ra"]>;
+  def X2_H  : RISCVReg<2, "x2", ["sp"]>;
+  def X3_H  : RISCVReg<3, "x3", ["gp"]>;
+  def X4_H  : RISCVReg<4, "x4", ["tp"]>;
+  def X5_H  : RISCVReg<5, "x5", ["t0"]>;
+  def X6_H  : RISCVReg<6, "x6", ["t1"]>;
+  def X7_H  : RISCVReg<7, "x7", ["t2"]>;
   }
-  def X8  : RISCVReg<8, "x8", ["s0", "fp"]>, DwarfRegNum<[8]>;
-  def X9  : RISCVReg<9, "x9", ["s1"]>, DwarfRegNum<[9]>;
-  def X10 : RISCVReg<10,"x10", ["a0"]>, DwarfRegNum<[10]>;
-  def X11 : RISCVReg<11,"x11", ["a1"]>, DwarfRegNum<[11]>;
-  def X12 : RISCVReg<12,"x12", ["a2"]>, DwarfRegNum<[12]>;
-  def X13 : RISCVReg<13,"x13", ["a3"]>, DwarfRegNum<[13]>;
-  def X14 : RISCVReg<14,"x14", ["a4"]>, DwarfRegNum<[14]>;
-  def X15 : RISCVReg<15,"x15", ["a5"]>, DwarfRegNum<[15]>;
+  def X8_H  : RISCVReg<8, "x8", ["s0", "fp"]>;
+  def X9_H  : RISCVReg<9, "x9", ["s1"]>;
+  def X10_H : RISCVReg<10,"x10", ["a0"]>;
+  def X11_H : RISCVReg<11,"x11", ["a1"]>;
+  def X12_H : RISCVReg<12,"x12", ["a2"]>;
+  def X13_H : RISCVReg<13,"x13", ["a3"]>;
+  def X14_H : RISCVReg<14,"x14", ["a4"]>;
+  def X15_H : RISCVReg<15,"x15", ["a5"]>;
   let CostPerUse = [0, 1] in {
-  def X16 : RISCVReg<16,"x16", ["a6"]>, DwarfRegNum<[16]>;
-  def X17 : RISCVReg<17,"x17", ["a7"]>, DwarfRegNum<[17]>;
-  def X18 : RISCVReg<18,"x18", ["s2"]>, DwarfRegNum<[18]>;
-  def X19 : RISCVReg<19,"x19", ["s3"]>, DwarfRegNum<[19]>;
-  def X20 : RISCVReg<20,"x20", ["s4"]>, DwarfRegNum<[20]>;
-  def X21 : RISCVReg<21,"x21", ["s5"]>, DwarfRegNum<[21]>;
-  def X22 : RISCVReg<22,"x22", ["s6"]>, DwarfRegNum<[22]>;
-  def X23 : RISCVReg<23,"x23", ["s7"]>, DwarfRegNum<[23]>;
-  def X24 : RISCVReg<24,"x24", ["s8"]>, DwarfRegNum<[24]>;
-  def X25 : RISCVReg<25,"x25", ["s9"]>, DwarfRegNum<[25]>;
-  def X26 : RISCVReg<26,"x26", ["s10"]>, DwarfRegNum<[26]>;
-  def X27 : RISCVReg<27,"x27", ["s11"]>, DwarfRegNum<[27]>;
-  def X28 : RISCVReg<28,"x28", ["t3"]>, DwarfRegNum<[28]>;
-  def X29 : RISCVReg<29,"x29", ["t4"]>, DwarfRegNum<[29]>;
-  def X30 : RISCVReg<30,"x30", ["t5"]>, DwarfRegNum<[30]>;
-  def X31 : RISCVReg<31,"x31", ["t6"]>, DwarfRegNum<[31]>;
+  def X16_H : RISCVReg<16,"x16", ["a6"]>;
+  def X17_H : RISCVReg<17,"x17", ["a7"]>;
+  def X18_H : RISCVReg<18,"x18", ["s2"]>;
+  def X19_H : RISCVReg<19,"x19", ["s3"]>;
+  def X20_H : RISCVReg<20,"x20", ["s4"]>;
+  def X21_H : RISCVReg<21,"x21", ["s5"]>;
+  def X22_H : RISCVReg<22,"x22", ["s6"]>;
+  def X23_H : RISCVReg<23,"x23", ["s7"]>;
+  def X24_H : RISCVReg<24,"x24", ["s8"]>;
+  def X25_H : RISCVReg<25,"x25", ["s9"]>;
+  def X26_H : RISCVReg<26,"x26", ["s10"]>;
+  def X27_H : RISCVReg<27,"x27", ["s11"]>;
+  def X28_H : RISCVReg<28,"x28", ["t3"]>;
+  def X29_H : RISCVReg<29,"x29", ["t4"]>;
+  def X30_H : RISCVReg<30,"x30", ["t5"]>;
+  def X31_H : RISCVReg<31,"x31", ["t6"]>;
+  }
+
+  let SubRegIndices = [sub_16] in {
+  let isConstant = true in
+  def X0  : RISCVRegWithSubRegs<0, "x0", [X0_H], ["zero"]>, DwarfRegNum<[0]>;
+  let CostPerUse = [0, 1] in {
+  def X1  : RISCVRegWithSubRegs<1, "x1", [X1_H], ["ra"]>, DwarfRegNum<[1]>;
+  def X2  : RISCVRegWithSubRegs<2, "x2", [X2_H], ["sp"]>, DwarfRegNum<[2]>;
+  def X3  : RISCVRegWithSubRegs<3, "x3", [X3_H], ["gp"]>, DwarfRegNum<[3]>;
+  def X4  : RISCVRegWithSubRegs<4, "x4", [X4_H], ["tp"]>, DwarfRegNum<[4]>;
+  def X5  : RISCVRegWithSubRegs<5, "x5", [X5_H], ["t0"]>, DwarfRegNum<[5]>;
+  def X6  : RISCVRegWithSubRegs<6, "x6", [X6_H], ["t1"]>, DwarfRegNum<[6]>;
+  def X7  : RISCVRegWithSubRegs<7, "x7", [X7_H], ["t2"]>, DwarfRegNum<[7]>;
+  }
+  def X8  : RISCVRegWithSubRegs<8, "x8", [X8_H], ["s0", "fp"]>, DwarfRegNum<[8]>;
+  def X9  : RISCVRegWithSubRegs<9, "x9", [X9_H], ["s1"]>, DwarfRegNum<[9]>;
+  def X10 : RISCVRegWithSubRegs<10,"x10", [X10_H], ["a0"]>, DwarfRegNum<[10]>;
+  def X11 : RISCVRegWithSubRegs<11,"x11", [X11_H], ["a1"]>, DwarfRegNum<[11]>;
+  def X12 : RISCVRegWithSubRegs<12,"x12", [X12_H], ["a2"]>, DwarfRegNum<[12]>;
+  def X13 : RISCVRegWithSubRegs<13,"x13", [X13_H], ["a3"]>, DwarfRegNum<[13]>;
+  def X14 : RISCVRegWithSubRegs<14,"x14", [X14_H], ["a4"]>, DwarfRegNum<[14]>;
+  def X15 : RISCVRegWithSubRegs<15,"x15", [X15_H], ["a5"]>, DwarfRegNum<[15]>;
+  let CostPerUse = [0, 1] in {
+  def X16 : RISCVRegWithSubRegs<16,"x16", [X16_H], ["a6"]>, DwarfRegNum<[16]>;
+  def X17 : RISCVRegWithSubRegs<17,"x17", [X17_H], ["a7"]>, DwarfRegNum<[17]>;
+  def X18 : RISCVRegWithSubRegs<18,"x18", [X18_H], ["s2"]>, DwarfRegNum<[18]>;
+  def X19 : RISCVRegWithSubRegs<19,"x19", [X19_H], ["s3"]>, DwarfRegNum<[19]>;
+  def X20 : RISCVRegWithSubRegs<20,"x20", [X20_H], ["s4"]>, DwarfRegNum<[20]>;
+  def X21 : RISCVRegWithSubRegs<21,"x21", [X21_H], ["s5"]>, DwarfRegNum<[21]>;
+  def X22 : RISCVRegWithSubRegs<22,"x22", [X22_H], ["s6"]>, DwarfRegNum<[22]>;
+  def X23 : RISCVRegWithSubRegs<23,"x23", [X23_H], ["s7"]>, DwarfRegNum<[23]>;
+  def X24 : RISCVRegWithSubRegs<24,"x24", [X24_H], ["s8"]>, DwarfRegNum<[24]>;
+  def X25 : RISCVRegWithSubRegs<25,"x25", [X25_H], ["s9"]>, DwarfRegNum<[25]>;
+  def X26 : RISCVRegWithSubRegs<26,"x26", [X26_H], ["s10"]>, DwarfRegNum<[26]>;
+  def X27 : RISCVRegWithSubRegs<27,"x27", [X27_H], ["s11"]>, DwarfRegNum<[27]>;
+  def X28 : RISCVRegWithSubRegs<28,"x28", [X28_H], ["t3"]>, DwarfRegNum<[28]>;
+  def X29 : RISCVRegWithSubRegs<29,"x29", [X29_H], ["t4"]>, DwarfRegNum<[29]>;
+  def X30 : RISCVRegWithSubRegs<30,"x30", [X30_H], ["t5"]>, DwarfRegNum<[30]>;
+  def X31 : RISCVRegWithSubRegs<31,"x31", [X31_H], ["t6"]>, DwarfRegNum<[31]>;
+  }
   }
 }
 
@@ -565,8 +607,17 @@ def VRM8NoV0 : VReg<VM8VTs, (sub VRM8, V0M8), 8>;
 
 def VMV0 : VReg<VMaskVTs, (add V0), 1>;
 
+// 16-bit GPR sub-register class used by Zhinx instructions.
+def GPRF16 : RISCVRegisterClass<[f16], 16, (add (sequence "X%u_H", 10, 17),
+                                                (sequence "X%u_H", 5, 7),
+                                                (sequence "X%u_H", 28, 31),
+                                                (sequence "X%u_H", 8, 9),
+                                                (sequence "X%u_H", 18, 27),
+                                                (sequence "X%u_H", 0, 4))>;
+def GPRF16C : RISCVRegisterClass<[f16], 16, (add (sequence "X%u_H", 10, 15),
+                                                 (sequence "X%u_H", 8, 9))>;
+
 let RegInfos = XLenRI in {
-def GPRF16  : RISCVRegisterClass<[f16], 16, (add GPR)>;
 def GPRF32  : RISCVRegisterClass<[f32], 32, (add GPR)>;
 } // RegInfos = XLenRI
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index e041854ee8fd6..c042782389f18 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1163,9 +1163,47 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
       Dst->getScalarSizeInBits() > ST->getELen())
     return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 
+  int ISD = TLI->InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
   std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Src);
   std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Dst);
 
+  // Handle i1 source and dest cases *before* calling logic in BasicTTI.
+  // The shared implementation doesn't model vector widening during legalization
+  // and instead assumes scalarization.  In order to scalarize an <N x i1>
+  // vector, we need to extend/trunc to/from i8.  If we don't special case
+  // this, we can get an infinite recursion cycle.
+  switch (ISD) {
+  default:
+    break;
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND:
+    if (Src->getScalarSizeInBits() == 1) {
+      // We do not use vsext/vzext to extend from mask vector.
+      // Instead we use the following instructions to extend from mask vector:
+      // vmv.v.i v8, 0
+      // vmerge.vim v8, v8, -1, v0 (repeated per split)
+      return getRISCVInstructionCost(RISCV::VMV_V_I, DstLT.second, CostKind) +
+             DstLT.first * getRISCVInstructionCost(RISCV::VMERGE_VIM,
+                                                   DstLT.second, CostKind) +
+             DstLT.first - 1;
+    }
+    break;
+  case ISD::TRUNCATE:
+    if (Dst->getScalarSizeInBits() == 1) {
+      // We do not use several vncvt to truncate to mask vector. So we could
+      // not use PowDiff to calculate it.
+      // Instead we use the following instructions to truncate to mask vector:
+      // vand.vi v8, v8, 1
+      // vmsne.vi v0, v8, 0
+      return SrcLT.first *
+                 getRISCVInstructionCost({RISCV::VAND_VI, RISCV::VMSNE_VI},
+                                         SrcLT.second, CostKind) +
+             SrcLT.first - 1;
+    }
+    break;
+  };
+
   // Our actual lowering for the case where a wider legal type is available
   // uses promotion to the wider type.  This is reflected in the result of
   // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
@@ -1181,22 +1219,11 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   // The split cost is handled by the base getCastInstrCost
   assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
 
-  int ISD = TLI->InstructionOpcodeToISD(Opcode);
-  assert(ISD && "Invalid opcode");
-
   int PowDiff = (int)Log2_32(DstLT.second.getScalarSizeInBits()) -
                 (int)Log2_32(SrcLT.second.getScalarSizeInBits());
   switch (ISD) {
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND: {
-    if (Src->getScalarSizeInBits() == 1) {
-      // We do not use vsext/vzext to extend from mask vector.
-      // Instead we use the following instructions to extend from mask vector:
-      // vmv.v.i v8, 0
-      // vmerge.vim v8, v8, -1, v0
-      return getRISCVInstructionCost({RISCV::VMV_V_I, RISCV::VMERGE_VIM},
-                                     DstLT.second, CostKind);
-    }
     if ((PowDiff < 1) || (PowDiff > 3))
       return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
     unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
@@ -1206,16 +1233,6 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
     return getRISCVInstructionCost(Op, DstLT.second, CostKind);
   }
   case ISD::TRUNCATE:
-    if (Dst->getScalarSizeInBits() == 1) {
-      // We do not use several vncvt to truncate to mask vector. So we could
-      // not use PowDiff to calculate it.
-      // Instead we use the following instructions to truncate to mask vector:
-      // vand.vi v8, v8, 1
-      // vmsne.vi v0, v8, 0
-      return getRISCVInstructionCost({RISCV::VAND_VI, RISCV::VMSNE_VI},
-                                     SrcLT.second, CostKind);
-    }
-    [[fallthrough]];
   case ISD::FP_EXTEND:
   case ISD::FP_ROUND: {
     // Counts of narrow/widen instructions.
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 86be79cbb5e7f..415b5d99695f0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -188,6 +188,21 @@ bool isConvergenceIntrinsic(const Instruction *I) {
          II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
          II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor;
 }
+
+bool allowEmitFakeUse(const Value *Arg) {
+  if (const auto *II = dyn_cast<IntrinsicInst>(Arg))
+    if (Function *F = II->getCalledFunction())
+      if (F->getName().starts_with("llvm.spv."))
+        return false;
+  if (dyn_cast<AtomicCmpXchgInst>(Arg) || dyn_cast<InsertValueInst>(Arg) ||
+      dyn_cast<UndefValue>(Arg))
+    return false;
+  if (const auto *LI = dyn_cast<LoadInst>(Arg))
+    if (LI->getType()->isAggregateType())
+      return false;
+  return true;
+}
+
 } // namespace
 
 char SPIRVEmitIntrinsics::ID = 0;
@@ -283,8 +298,20 @@ static inline Type *reconstructType(SPIRVGlobalRegistry *GR, Value *Op) {
 void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty,
                                           Value *Arg) {
   Value *OfType = PoisonValue::get(Ty);
-  CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,
-                                       {Arg->getType()}, OfType, Arg, {}, B);
+  CallInst *AssignCI = nullptr;
+  if (Arg->getType()->isAggregateType() && Ty->isAggregateType() &&
+      allowEmitFakeUse(Arg)) {
+    LLVMContext &Ctx = Arg->getContext();
+    SmallVector<Metadata *, 2> ArgMDs{
+        MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)),
+        MDString::get(Ctx, Arg->getName())};
+    B.CreateIntrinsic(Intrinsic::spv_value_md, {},
+                      {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});
+    AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg});
+  } else {
+    AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},
+                               OfType, Arg, {}, B);
+  }
   GR->addAssignPtrTypeInstr(Arg, AssignCI);
 }
 
@@ -1268,6 +1295,8 @@ Instruction *SPIRVEmitIntrinsics::visitInsertValueInst(InsertValueInst &I) {
 }
 
 Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) {
+  if (I.getAggregateOperand()->getType()->isAggregateType())
+    return &I;
   IRBuilder<> B(I.getParent());
   B.SetInsertPoint(&I);
   SmallVector<Value *> Args;
@@ -1534,7 +1563,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I,
       I->setOperand(OpNo, NewOp);
     }
   }
-  if (I->hasName()) {
+  if (I->hasName() && !I->getType()->isAggregateType()) {
     reportFatalOnTokenType(I);
     setInsertPointAfterDef(B, I);
     std::vector<Value *> Args = {I};
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
index b78f1c3f060a2..f95f0d2988be2 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
@@ -73,6 +73,22 @@ enum BaseTypeAttributeEncoding {
   UnsignedChar = 7
 };
 
+enum SourceLanguage {
+  Unknown = 0,
+  ESSL = 1,
+  GLSL = 2,
+  OpenCL_C = 3,
+  OpenCL_CPP = 4,
+  HLSL = 5,
+  CPP_for_OpenCL = 6,
+  SYCL = 7,
+  HERO_C = 8,
+  NZSL = 9,
+  WGSL = 10,
+  Slang = 11,
+  Zig = 12
+};
+
 bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
   // If this MachineFunction doesn't have any BB repeat procedure
   // for the next
@@ -83,8 +99,8 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
 
   // Required variables to get from metadata search
   LLVMContext *Context;
-  SmallString<128> FilePath;
-  unsigned SourceLanguage = 0;
+  SmallVector<SmallString<128>> FilePaths;
+  SmallVector<int64_t> LLVMSourceLanguages;
   int64_t DwarfVersion = 0;
   int64_t DebugInfoVersion = 0;
   SmallPtrSet<DIBasicType *, 12> BasicTypes;
@@ -101,9 +117,10 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
     for (const auto *Op : DbgCu->operands()) {
       if (const auto *CompileUnit = dyn_cast<DICompileUnit>(Op)) {
         DIFile *File = CompileUnit->getFile();
-        sys::path::append(FilePath, File->getDirectory(), File->getFilename());
-        SourceLanguage = CompileUnit->getSourceLanguage();
-        break;
+        FilePaths.emplace_back();
+        sys::path::append(FilePaths.back(), File->getDirectory(),
+                          File->getFilename());
+        LLVMSourceLanguages.push_back(CompileUnit->getSourceLanguage());
       }
     }
     const NamedMDNode *ModuleFlags = M->getNamedMetadata("llvm.module.flags");
@@ -160,9 +177,6 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
       return StrReg;
     };
 
-    // Emit OpString with FilePath which is required by DebugSource
-    const Register FilePathStrReg = EmitOpString(FilePath);
-
     const SPIRVType *VoidTy =
         GR->getOrCreateSPIRVType(Type::getVoidTy(*Context), MIRBuilder);
 
@@ -187,27 +201,54 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
           return InstReg;
         };
 
-    // Emit DebugSource which is required by DebugCompilationUnit
-    const Register DebugSourceResIdReg = EmitDIInstruction(
-        SPIRV::NonSemanticExtInst::DebugSource, {FilePathStrReg});
-
     const SPIRVType *I32Ty =
         GR->getOrCreateSPIRVType(Type::getInt32Ty(*Context), MIRBuilder);
 
-    // Convert DwarfVersion, DebugInfo and SourceLanguage integers to OpConstant
-    // instructions required by DebugCompilationUnit
     const Register DwarfVersionReg =
         GR->buildConstantInt(DwarfVersion, MIRBuilder, I32Ty, false);
     const Register DebugInfoVersionReg =
         GR->buildConstantInt(DebugInfoVersion, MIRBuilder, I32Ty, false);
-    const Register SourceLanguageReg =
-        GR->buildConstantInt(SourceLanguage, MIRBuilder, I32Ty, false);
-
-    [[maybe_unused]]
-    const Register DebugCompUnitResIdReg =
-        EmitDIInstruction(SPIRV::NonSemanticExtInst::DebugCompilationUnit,
-                          {DebugInfoVersionReg, DwarfVersionReg,
-                           DebugSourceResIdReg, SourceLanguageReg});
+
+    for (unsigned Idx = 0; Idx < LLVMSourceLanguages.size(); ++Idx) {
+      const Register FilePathStrReg = EmitOpString(FilePaths[Idx]);
+
+      const Register DebugSourceResIdReg = EmitDIInstruction(
+          SPIRV::NonSemanticExtInst::DebugSource, {FilePathStrReg});
+
+      SourceLanguage SpirvSourceLanguage = SourceLanguage::Unknown;
+      switch (LLVMSourceLanguages[Idx]) {
+      case dwarf::DW_LANG_OpenCL:
+        SpirvSourceLanguage = SourceLanguage::OpenCL_C;
+        break;
+      case dwarf::DW_LANG_OpenCL_CPP:
+        SpirvSourceLanguage = SourceLanguage::OpenCL_CPP;
+        break;
+      case dwarf::DW_LANG_CPP_for_OpenCL:
+        SpirvSourceLanguage = SourceLanguage::CPP_for_OpenCL;
+        break;
+      case dwarf::DW_LANG_GLSL:
+        SpirvSourceLanguage = SourceLanguage::GLSL;
+        break;
+      case dwarf::DW_LANG_HLSL:
+        SpirvSourceLanguage = SourceLanguage::HLSL;
+        break;
+      case dwarf::DW_LANG_SYCL:
+        SpirvSourceLanguage = SourceLanguage::SYCL;
+        break;
+      case dwarf::DW_LANG_Zig:
+        SpirvSourceLanguage = SourceLanguage::Zig;
+        break;
+      }
+
+      const Register SourceLanguageReg =
+          GR->buildConstantInt(SpirvSourceLanguage, MIRBuilder, I32Ty, false);
+
+      [[maybe_unused]]
+      const Register DebugCompUnitResIdReg =
+          EmitDIInstruction(SPIRV::NonSemanticExtInst::DebugCompilationUnit,
+                            {DebugInfoVersionReg, DwarfVersionReg,
+                             DebugSourceResIdReg, SourceLanguageReg});
+    }
 
     // We aren't extracting any DebugInfoFlags now so we
     // emitting zero to use as <id>Flags argument for DebugBasicType
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index cad2bf96adf33..92f95418624fe 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -55,6 +55,8 @@ class SPIRVGlobalRegistry {
   // created during substitution of aggregate arguments
   // (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`)
   DenseMap<Value *, Type *> MutatedAggRet;
+  // map an instruction to its value's attributes (type, name)
+  DenseMap<MachineInstr *, std::pair<Type *, std::string>> ValueAttrs;
 
   // Look for an equivalent of the newType in the map. Return the equivalent
   // if it's found, otherwise insert newType to the map and return the type.
@@ -188,6 +190,21 @@ class SPIRVGlobalRegistry {
     return It == MutatedAggRet.end() ? nullptr : It->second;
   }
 
+  // A registry of value's attributes (type, name)
+  // - Add a record.
+  void addValueAttrs(MachineInstr *Key, std::pair<Type *, std::string> Val) {
+    ValueAttrs[Key] = Val;
+  }
+  // - Find a record.
+  bool findValueAttrs(const MachineInstr *Key, Type *&Ty, StringRef &Name) {
+    auto It = ValueAttrs.find(Key);
+    if (It == ValueAttrs.end())
+      return false;
+    Ty = It->second.first;
+    Name = It->second.second;
+    return true;
+  }
+
   // Deduced element types of untyped pointers and composites:
   // - Add a record to the map of deduced element types.
   void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 51bacb00b1c51..fe45be4daba65 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -519,8 +519,8 @@ def OpMatrixTimesMatrix: BinOp<"OpMatrixTimesMatrix", 146>;
 def OpOuterProduct: BinOp<"OpOuterProduct", 147>;
 def OpDot: BinOp<"OpDot", 148>;
 
-def OpIAddCarry: BinOpTyped<"OpIAddCarry", 149, iID, addc>;
-def OpISubBorrow: BinOpTyped<"OpISubBorrow", 150, iID, subc>;
+defm OpIAddCarry: BinOpTypedGen<"OpIAddCarry", 149, addc, 0, 1>;
+defm OpISubBorrow: BinOpTypedGen<"OpISubBorrow", 150, subc, 0, 1>;
 def OpUMulExtended: BinOp<"OpUMulExtended", 151>;
 def OpSMulExtended: BinOp<"OpSMulExtended", 152>;
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index e475810f92f71..2f7efbdc81f84 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -169,6 +169,9 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectFloatDot(Register ResVReg, const SPIRVType *ResType,
                       MachineInstr &I) const;
 
+  bool selectOverflowArith(Register ResVReg, const SPIRVType *ResType,
+                           MachineInstr &I, unsigned Opcode) const;
+
   bool selectIntegerDot(Register ResVReg, const SPIRVType *ResType,
                         MachineInstr &I) const;
 
@@ -386,11 +389,22 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) {
   return false;
 }
 
+static bool mayApplyGenericSelection(unsigned Opcode) {
+  switch (Opcode) {
+  case TargetOpcode::G_CONSTANT:
+    return false;
+  case TargetOpcode::G_SADDO:
+  case TargetOpcode::G_SSUBO:
+    return true;
+  }
+  return isTypeFoldingSupported(Opcode);
+}
+
 bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
                                          const SPIRVType *ResType,
                                          MachineInstr &I) const {
   const unsigned Opcode = I.getOpcode();
-  if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT)
+  if (mayApplyGenericSelection(Opcode))
     return selectImpl(I, *CoverageInfo);
   switch (Opcode) {
   case TargetOpcode::G_CONSTANT:
@@ -526,6 +540,8 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
     return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin);
   case TargetOpcode::G_FATAN:
     return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan);
+  case TargetOpcode::G_FATAN2:
+    return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2);
   case TargetOpcode::G_FCOSH:
     return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh);
   case TargetOpcode::G_FSINH:
@@ -567,6 +583,21 @@ bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
   case TargetOpcode::G_USUBSAT:
     return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat);
 
+  case TargetOpcode::G_UADDO:
+    return selectOverflowArith(ResVReg, ResType, I,
+                               ResType->getOpcode() == SPIRV::OpTypeVector
+                                   ? SPIRV::OpIAddCarryV
+                                   : SPIRV::OpIAddCarryS);
+  case TargetOpcode::G_USUBO:
+    return selectOverflowArith(ResVReg, ResType, I,
+                               ResType->getOpcode() == SPIRV::OpTypeVector
+                                   ? SPIRV::OpISubBorrowV
+                                   : SPIRV::OpISubBorrowS);
+  case TargetOpcode::G_UMULO:
+    return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended);
+  case TargetOpcode::G_SMULO:
+    return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended);
+
   case TargetOpcode::G_SEXT:
     return selectExt(ResVReg, ResType, I, true);
   case TargetOpcode::G_ANYEXT:
@@ -1056,6 +1087,71 @@ bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
       .constrainAllUses(TII, TRI, RBI);
 }
 
+bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg,
+                                                   const SPIRVType *ResType,
+                                                   MachineInstr &I,
+                                                   unsigned Opcode) const {
+  Type *ResTy = nullptr;
+  StringRef ResName;
+  if (!GR.findValueAttrs(&I, ResTy, ResName))
+    report_fatal_error(
+        "Not enough info to select the arithmetic with overflow instruction");
+  if (!ResTy || !ResTy->isStructTy())
+    report_fatal_error("Expect struct type result for the arithmetic "
+                       "with overflow instruction");
+  // "Result Type must be from OpTypeStruct. The struct must have two members,
+  // and the two members must be the same type."
+  Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0);
+  ResTy = StructType::create(SmallVector<Type *, 2>{ResElemTy, ResElemTy});
+  // Build SPIR-V types and constant(s) if needed.
+  MachineIRBuilder MIRBuilder(I);
+  SPIRVType *StructType = GR.getOrCreateSPIRVType(
+      ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
+  assert(I.getNumDefs() > 1 && "Not enought operands");
+  SPIRVType *BoolType = GR.getOrCreateSPIRVBoolType(I, TII);
+  unsigned N = GR.getScalarOrVectorComponentCount(ResType);
+  if (N > 1)
+    BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII);
+  Register BoolTypeReg = GR.getSPIRVTypeID(BoolType);
+  Register ZeroReg = buildZerosVal(ResType, I);
+  // A new virtual register to store the result struct.
+  Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+  MRI->setRegClass(StructVReg, &SPIRV::IDRegClass);
+  // Build the result name if needed.
+  if (ResName.size() > 0)
+    buildOpName(StructVReg, ResName, MIRBuilder);
+  // Build the arithmetic with overflow instruction.
+  MachineBasicBlock &BB = *I.getParent();
+  auto MIB =
+      BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode))
+          .addDef(StructVReg)
+          .addUse(GR.getSPIRVTypeID(StructType));
+  for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i)
+    MIB.addUse(I.getOperand(i).getReg());
+  bool Status = MIB.constrainAllUses(TII, TRI, RBI);
+  // Build instructions to extract fields of the instruction's result.
+  // A new virtual register to store the higher part of the result struct.
+  Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64));
+  MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass);
+  for (unsigned i = 0; i < I.getNumDefs(); ++i) {
+    auto MIB =
+        BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
+            .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg())
+            .addUse(GR.getSPIRVTypeID(ResType))
+            .addUse(StructVReg)
+            .addImm(i);
+    Status &= MIB.constrainAllUses(TII, TRI, RBI);
+  }
+  // Build boolean value from the higher part.
+  Status &= BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
+                .addDef(I.getOperand(1).getReg())
+                .addUse(BoolTypeReg)
+                .addUse(HigherVReg)
+                .addUse(ZeroReg)
+                .constrainAllUses(TII, TRI, RBI);
+  return Status;
+}
+
 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
                                                    const SPIRVType *ResType,
                                                    MachineInstr &I) const {
@@ -2460,6 +2556,9 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
   }
   case Intrinsic::spv_step:
     return selectStep(ResVReg, ResType, I);
+  case Intrinsic::spv_value_md:
+    // ignore the intrinsic
+    break;
   default: {
     std::string DiagMsg;
     raw_string_ostream OS(DiagMsg);
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 9fe4d8a16bc32..460f0127d4ffc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -287,7 +287,8 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
   // TODO: add proper legalization rules.
   getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG).alwaysLegal();
 
-  getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_SMULO, G_UMULO})
+  getActionDefinitionsBuilder(
+      {G_UADDO, G_SADDO, G_USUBO, G_SSUBO, G_UMULO, G_SMULO})
       .alwaysLegal();
 
   // FP conversions.
@@ -320,6 +321,7 @@ SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
                                G_FACOS,
                                G_FASIN,
                                G_FATAN,
+                               G_FATAN2,
                                G_FCOSH,
                                G_FSINH,
                                G_FTANH,
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index cd0aff1a51843..42f3ded336f95 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -376,7 +376,13 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
       .addUse(NewReg)
       .addUse(GR->getSPIRVTypeID(SpvType))
       .setMIFlags(Flags);
-  Def->getOperand(0).setReg(NewReg);
+  for (unsigned I = 0, E = Def->getNumDefs(); I != E; ++I) {
+    MachineOperand &MO = Def->getOperand(I);
+    if (MO.getReg() == Reg) {
+      MO.setReg(NewReg);
+      break;
+    }
+  }
   return NewReg;
 }
 
@@ -460,6 +466,25 @@ generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,
             Def->getOpcode() != SPIRV::ASSIGN_TYPE)
           insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());
         ToErase.push_back(&MI);
+      } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {
+        MachineInstr *MdMI = MI.getPrevNode();
+        if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) {
+          // It's an internal service info from before IRTranslator passes.
+          MachineInstr *Def = getVRegDef(MRI, MI.getOperand(0).getReg());
+          for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)
+            if (getVRegDef(MRI, MI.getOperand(I).getReg()) != Def)
+              Def = nullptr;
+          if (Def) {
+            const MDNode *MD = MdMI->getOperand(1).getMetadata();
+            StringRef ValueName =
+                cast<MDString>(MD->getOperand(1))->getString();
+            const MDNode *TypeMD = cast<MDNode>(MD->getOperand(0));
+            Type *ValueTy = getMDOperandAsType(TypeMD, 0);
+            GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName.str()));
+          }
+          ToErase.push_back(MdMI);
+        }
+        ToErase.push_back(&MI);
       } else if (MIOp == TargetOpcode::G_CONSTANT ||
                  MIOp == TargetOpcode::G_FCONSTANT ||
                  MIOp == TargetOpcode::G_BUILD_VECTOR) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index eb5139ac5383a..1872b238d1077 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -342,30 +342,6 @@ static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
   FSHIntrinsic->setCalledFunction(FSHFunc);
 }
 
-static void buildUMulWithOverflowFunc(Function *UMulFunc) {
-  // The function body is already created.
-  if (!UMulFunc->empty())
-    return;
-
-  BasicBlock *EntryBB = BasicBlock::Create(UMulFunc->getParent()->getContext(),
-                                           "entry", UMulFunc);
-  IRBuilder<> IRB(EntryBB);
-  // Build the actual unsigned multiplication logic with the overflow
-  // indication. Do unsigned multiplication Mul = A * B. Then check
-  // if unsigned division Div = Mul / A is not equal to B. If so,
-  // then overflow has happened.
-  Value *Mul = IRB.CreateNUWMul(UMulFunc->getArg(0), UMulFunc->getArg(1));
-  Value *Div = IRB.CreateUDiv(Mul, UMulFunc->getArg(0));
-  Value *Overflow = IRB.CreateICmpNE(UMulFunc->getArg(0), Div);
-
-  // umul.with.overflow intrinsic return a structure, where the first element
-  // is the multiplication result, and the second is an overflow bit.
-  Type *StructTy = UMulFunc->getReturnType();
-  Value *Agg = IRB.CreateInsertValue(PoisonValue::get(StructTy), Mul, {0});
-  Value *Res = IRB.CreateInsertValue(Agg, Overflow, {1});
-  IRB.CreateRet(Res);
-}
-
 static void lowerExpectAssume(IntrinsicInst *II) {
   // If we cannot use the SPV_KHR_expect_assume extension, then we need to
   // ignore the intrinsic and move on. It should be removed later on by LLVM.
@@ -407,20 +383,6 @@ static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID,
   return true;
 }
 
-static void lowerUMulWithOverflow(IntrinsicInst *UMulIntrinsic) {
-  // Get a separate function - otherwise, we'd have to rework the CFG of the
-  // current one. Then simply replace the intrinsic uses with a call to the new
-  // function.
-  Module *M = UMulIntrinsic->getModule();
-  FunctionType *UMulFuncTy = UMulIntrinsic->getFunctionType();
-  Type *FSHLRetTy = UMulFuncTy->getReturnType();
-  const std::string FuncName = lowerLLVMIntrinsicName(UMulIntrinsic);
-  Function *UMulFunc =
-      getOrCreateFunction(M, FSHLRetTy, UMulFuncTy->params(), FuncName);
-  buildUMulWithOverflowFunc(UMulFunc);
-  UMulIntrinsic->setCalledFunction(UMulFunc);
-}
-
 // Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
 // or calls to proper generated functions. Returns True if F was modified.
 bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
@@ -444,10 +406,6 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
         lowerFunnelShifts(II);
         Changed = true;
         break;
-      case Intrinsic::umul_with_overflow:
-        lowerUMulWithOverflow(II);
-        Changed = true;
-        break;
       case Intrinsic::assume:
       case Intrinsic::expect: {
         const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(*F);
@@ -478,9 +436,13 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
 // noted in 'spv.cloned_funcs' metadata for later restoration.
 Function *
 SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
+  bool IsRetAggr = F->getReturnType()->isAggregateType();
+  // Allow intrinsics with aggregate return type to reach GlobalISel
+  if (F->isIntrinsic() && IsRetAggr)
+    return F;
+
   IRBuilder<> B(F->getContext());
 
-  bool IsRetAggr = F->getReturnType()->isAggregateType();
   bool HasAggrArg =
       std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) {
         return Arg.getType()->isAggregateType();
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index a8016d42b0154..3640188670d15 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -45,10 +45,7 @@ static uint32_t convertCharsToWord(const StringRef &Str, unsigned i) {
 }
 
 // Get length including padding and null terminator.
-static size_t getPaddedLen(const StringRef &Str) {
-  const size_t Len = Str.size() + 1;
-  return (Len % 4 == 0) ? Len : Len + (4 - (Len % 4));
-}
+static size_t getPaddedLen(const StringRef &Str) { return Str.size() + 4 & ~3; }
 
 void addStringImm(const StringRef &Str, MCInst &Inst) {
   const size_t PaddedLen = getPaddedLen(Str);
@@ -617,4 +614,11 @@ bool sortBlocks(Function &F) {
   return Modified;
 }
 
+MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg) {
+  MachineInstr *MaybeDef = MRI.getVRegDef(Reg);
+  if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)
+    MaybeDef = MRI.getVRegDef(MaybeDef->getOperand(1).getReg());
+  return MaybeDef;
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index 7c7616000d22b..0d9b238db1403 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -315,5 +315,7 @@ inline const Type *unifyPtrType(const Type *Ty) {
   return toTypedPointer(const_cast<Type *>(Ty));
 }
 
+MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg);
+
 } // namespace llvm
 #endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 129fdaf37fc0d..95db5500b0e1b 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -1255,7 +1255,7 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
 
   void onEndOfFunction(SMLoc ErrorLoc) {
     if (!SkipTypeCheck)
-      TC.endOfFunction(ErrorLoc);
+      TC.endOfFunction(ErrorLoc, true);
     // Reset the type checker state.
     TC.clear();
   }
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
index 8b1e1dca4f847..845bf3976c22b 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.cpp
@@ -33,6 +33,7 @@
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/SourceMgr.h"
+#include <sstream>
 
 using namespace llvm;
 
@@ -59,14 +60,7 @@ void WebAssemblyAsmTypeCheck::localDecl(
 }
 
 void WebAssemblyAsmTypeCheck::dumpTypeStack(Twine Msg) {
-  LLVM_DEBUG({
-    std::string s;
-    for (auto VT : Stack) {
-      s += WebAssembly::typeToString(VT);
-      s += " ";
-    }
-    dbgs() << Msg << s << '\n';
-  });
+  LLVM_DEBUG({ dbgs() << Msg << getTypesString(Stack, 0) << "\n"; });
 }
 
 bool WebAssemblyAsmTypeCheck::typeError(SMLoc ErrorLoc, const Twine &Msg) {
@@ -77,34 +71,124 @@ bool WebAssemblyAsmTypeCheck::typeError(SMLoc ErrorLoc, const Twine &Msg) {
   return Parser.Error(ErrorLoc, Msg);
 }
 
-bool WebAssemblyAsmTypeCheck::popType(SMLoc ErrorLoc,
-                                      std::optional<wasm::ValType> EVT) {
-  if (Stack.empty()) {
-    return typeError(ErrorLoc,
-                     EVT ? StringRef("empty stack while popping ") +
-                               WebAssembly::typeToString(*EVT)
-                         : StringRef("empty stack while popping value"));
+bool WebAssemblyAsmTypeCheck::match(StackType TypeA, StackType TypeB) {
+  if (TypeA == TypeB)
+    return false;
+  if (std::get_if<Any>(&TypeA) || std::get_if<Any>(&TypeB))
+    return false;
+
+  if (std::get_if<Ref>(&TypeB))
+    std::swap(TypeA, TypeB);
+  assert(std::get_if<wasm::ValType>(&TypeB));
+  if (std::get_if<Ref>(&TypeA) &&
+      WebAssembly::isRefType(std::get<wasm::ValType>(TypeB)))
+    return false;
+  return true;
+}
+
+std::string WebAssemblyAsmTypeCheck::getTypesString(ArrayRef<StackType> Types,
+                                                    size_t StartPos) {
+  SmallVector<std::string, 4> Reverse;
+  for (auto I = Types.size(); I > StartPos; I--) {
+    if (std::get_if<Any>(&Types[I - 1]))
+      Reverse.push_back("any");
+    else if (std::get_if<Ref>(&Types[I - 1]))
+      Reverse.push_back("ref");
+    else
+      Reverse.push_back(
+          WebAssembly::typeToString(std::get<wasm::ValType>(Types[I - 1])));
   }
-  auto PVT = Stack.pop_back_val();
-  if (EVT && *EVT != PVT) {
-    return typeError(ErrorLoc,
-                     StringRef("popped ") + WebAssembly::typeToString(PVT) +
-                         ", expected " + WebAssembly::typeToString(*EVT));
+
+  std::stringstream SS;
+  SS << "[";
+  bool First = true;
+  for (auto It = Reverse.rbegin(); It != Reverse.rend(); ++It) {
+    if (!First)
+      SS << ", ";
+    SS << *It;
+    First = false;
   }
-  return false;
+  SS << "]";
+  return SS.str();
 }
 
-bool WebAssemblyAsmTypeCheck::popRefType(SMLoc ErrorLoc) {
-  if (Stack.empty()) {
-    return typeError(ErrorLoc, StringRef("empty stack while popping reftype"));
-  }
-  auto PVT = Stack.pop_back_val();
-  if (!WebAssembly::isRefType(PVT)) {
-    return typeError(ErrorLoc, StringRef("popped ") +
-                                   WebAssembly::typeToString(PVT) +
-                                   ", expected reftype");
+SmallVector<WebAssemblyAsmTypeCheck::StackType, 4>
+WebAssemblyAsmTypeCheck::valTypeToStackType(ArrayRef<wasm::ValType> ValTypes) {
+  SmallVector<StackType, 4> Types(ValTypes.size());
+  std::transform(ValTypes.begin(), ValTypes.end(), Types.begin(),
+                 [](wasm::ValType Val) -> StackType { return Val; });
+  return Types;
+}
+
+bool WebAssemblyAsmTypeCheck::checkTypes(SMLoc ErrorLoc,
+                                         ArrayRef<wasm::ValType> ValTypes,
+                                         bool ExactMatch) {
+  return checkTypes(ErrorLoc, valTypeToStackType(ValTypes), ExactMatch);
+}
+
+bool WebAssemblyAsmTypeCheck::checkTypes(SMLoc ErrorLoc,
+                                         ArrayRef<StackType> Types,
+                                         bool ExactMatch) {
+  auto StackI = Stack.size();
+  auto TypeI = Types.size();
+  bool Error = false;
+  // Compare elements one by one from the stack top
+  for (; StackI > 0 && TypeI > 0; StackI--, TypeI--) {
+    if (match(Stack[StackI - 1], Types[TypeI - 1])) {
+      Error = true;
+      break;
+    }
   }
-  return false;
+  // Even if no match failure has happened in the loop above, if not all
+  // elements of Types has been matched, that means we don't have enough
+  // elements on the stack.
+  //
+  // Also, if not all elements of the Stack has been matched and when
+  // 'ExactMatch' is true, that means we have superfluous elements remaining on
+  // the stack (e.g. at the end of a function).
+  if (TypeI > 0 || (ExactMatch && StackI > 0))
+    Error = true;
+
+  if (!Error)
+    return false;
+
+  auto StackStartPos =
+      ExactMatch ? 0 : std::max(0, (int)Stack.size() - (int)Types.size());
+  return typeError(ErrorLoc, "type mismatch, expected " +
+                                 getTypesString(Types, 0) + " but got " +
+                                 getTypesString(Stack, StackStartPos));
+}
+
+bool WebAssemblyAsmTypeCheck::checkAndPopTypes(SMLoc ErrorLoc,
+                                               ArrayRef<wasm::ValType> ValTypes,
+                                               bool ExactMatch) {
+  return checkAndPopTypes(ErrorLoc, valTypeToStackType(ValTypes), ExactMatch);
+}
+
+bool WebAssemblyAsmTypeCheck::checkAndPopTypes(SMLoc ErrorLoc,
+                                               ArrayRef<StackType> Types,
+                                               bool ExactMatch) {
+  bool Error = checkTypes(ErrorLoc, Types, ExactMatch);
+  auto NumPops = std::min(Stack.size(), Types.size());
+  for (size_t I = 0, E = NumPops; I != E; I++)
+    Stack.pop_back();
+  return Error;
+}
+
+bool WebAssemblyAsmTypeCheck::popType(SMLoc ErrorLoc, StackType Type) {
+  return checkAndPopTypes(ErrorLoc, {Type}, false);
+}
+
+bool WebAssemblyAsmTypeCheck::popRefType(SMLoc ErrorLoc) {
+  return popType(ErrorLoc, Ref{});
+}
+
+bool WebAssemblyAsmTypeCheck::popAnyType(SMLoc ErrorLoc) {
+  return popType(ErrorLoc, Any{});
+}
+
+void WebAssemblyAsmTypeCheck::pushTypes(ArrayRef<wasm::ValType> ValTypes) {
+  Stack.append(valTypeToStackType(ValTypes));
 }
 
 bool WebAssemblyAsmTypeCheck::getLocal(SMLoc ErrorLoc, const MCOperand &LocalOp,
@@ -117,59 +201,29 @@ bool WebAssemblyAsmTypeCheck::getLocal(SMLoc ErrorLoc, const MCOperand &LocalOp,
   return false;
 }
 
-static std::optional<std::string>
-checkStackTop(const SmallVectorImpl<wasm::ValType> &ExpectedStackTop,
-              const SmallVectorImpl<wasm::ValType> &Got) {
-  for (size_t I = 0; I < ExpectedStackTop.size(); I++) {
-    auto EVT = ExpectedStackTop[I];
-    auto PVT = Got[Got.size() - ExpectedStackTop.size() + I];
-    if (PVT != EVT)
-      return std::string{"got "} + WebAssembly::typeToString(PVT) +
-             ", expected " + WebAssembly::typeToString(EVT);
-  }
-  return std::nullopt;
-}
-
 bool WebAssemblyAsmTypeCheck::checkBr(SMLoc ErrorLoc, size_t Level) {
   if (Level >= BrStack.size())
     return typeError(ErrorLoc,
                      StringRef("br: invalid depth ") + std::to_string(Level));
   const SmallVector<wasm::ValType, 4> &Expected =
       BrStack[BrStack.size() - Level - 1];
-  if (Expected.size() > Stack.size())
-    return typeError(ErrorLoc, "br: insufficient values on the type stack");
-  auto IsStackTopInvalid = checkStackTop(Expected, Stack);
-  if (IsStackTopInvalid)
-    return typeError(ErrorLoc, "br " + IsStackTopInvalid.value());
+  return checkTypes(ErrorLoc, Expected, false);
   return false;
 }
 
 bool WebAssemblyAsmTypeCheck::checkEnd(SMLoc ErrorLoc, bool PopVals) {
   if (!PopVals)
     BrStack.pop_back();
-  if (LastSig.Returns.size() > Stack.size())
-    return typeError(ErrorLoc, "end: insufficient values on the type stack");
 
-  if (PopVals) {
-    for (auto VT : llvm::reverse(LastSig.Returns)) {
-      if (popType(ErrorLoc, VT))
-        return true;
-    }
-    return false;
-  }
-
-  auto IsStackTopInvalid = checkStackTop(LastSig.Returns, Stack);
-  if (IsStackTopInvalid)
-    return typeError(ErrorLoc, "end " + IsStackTopInvalid.value());
-  return false;
+  if (PopVals)
+    return checkAndPopTypes(ErrorLoc, LastSig.Returns, false);
+  return checkTypes(ErrorLoc, LastSig.Returns, false);
 }
 
 bool WebAssemblyAsmTypeCheck::checkSig(SMLoc ErrorLoc,
                                        const wasm::WasmSignature &Sig) {
-  bool Error = false;
-  for (auto VT : llvm::reverse(Sig.Params))
-    Error |= popType(ErrorLoc, VT);
-  Stack.insert(Stack.end(), Sig.Returns.begin(), Sig.Returns.end());
+  bool Error = checkAndPopTypes(ErrorLoc, Sig.Params, false);
+  pushTypes(Sig.Returns);
   return Error;
 }
 
@@ -246,7 +300,7 @@ bool WebAssemblyAsmTypeCheck::getSignature(SMLoc ErrorLoc,
       TypeName = "tag";
       break;
     default:
-      return true;
+      llvm_unreachable("Signature symbol should either be a function or a tag");
     }
     return typeError(ErrorLoc, StringRef("symbol ") + WasmSym->getName() +
                                    ": missing ." + TypeName + "type");
@@ -254,15 +308,8 @@ bool WebAssemblyAsmTypeCheck::getSignature(SMLoc ErrorLoc,
   return false;
 }
 
-bool WebAssemblyAsmTypeCheck::endOfFunction(SMLoc ErrorLoc) {
-  bool Error = false;
-  // Check the return types.
-  for (auto RVT : llvm::reverse(ReturnTypes))
-    Error |= popType(ErrorLoc, RVT);
-  if (!Stack.empty()) {
-    return typeError(ErrorLoc, std::to_string(Stack.size()) +
-                                   " superfluous return values");
-  }
+bool WebAssemblyAsmTypeCheck::endOfFunction(SMLoc ErrorLoc, bool ExactMatch) {
+  bool Error = checkAndPopTypes(ErrorLoc, ReturnTypes, ExactMatch);
   Unreachable = true;
   return Error;
 }
@@ -276,7 +323,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
 
   if (Name == "local.get") {
     if (!getLocal(Operands[1]->getStartLoc(), Inst.getOperand(0), Type)) {
-      Stack.push_back(Type);
+      pushType(Type);
       return false;
     }
     return true;
@@ -291,7 +338,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   if (Name == "local.tee") {
     if (!getLocal(Operands[1]->getStartLoc(), Inst.getOperand(0), Type)) {
       bool Error = popType(ErrorLoc, Type);
-      Stack.push_back(Type);
+      pushType(Type);
       return Error;
     }
     return true;
@@ -299,7 +346,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
 
   if (Name == "global.get") {
     if (!getGlobal(Operands[1]->getStartLoc(), Inst.getOperand(0), Type)) {
-      Stack.push_back(Type);
+      pushType(Type);
       return false;
     }
     return true;
@@ -314,7 +361,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   if (Name == "table.get") {
     bool Error = popType(ErrorLoc, wasm::ValType::I32);
     if (!getTable(Operands[1]->getStartLoc(), Inst.getOperand(0), Type)) {
-      Stack.push_back(Type);
+      pushType(Type);
       return Error;
     }
     return true;
@@ -332,7 +379,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
 
   if (Name == "table.size") {
     bool Error = getTable(Operands[1]->getStartLoc(), Inst.getOperand(0), Type);
-    Stack.push_back(wasm::ValType::I32);
+    pushType(wasm::ValType::I32);
     return Error;
   }
 
@@ -342,7 +389,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
       Error |= popType(ErrorLoc, Type);
     else
       Error = true;
-    Stack.push_back(wasm::ValType::I32);
+    pushType(wasm::ValType::I32);
     return Error;
   }
 
@@ -381,7 +428,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   }
 
   if (Name == "drop") {
-    return popType(ErrorLoc, {});
+    return popType(ErrorLoc, Any{});
   }
 
   if (Name == "try" || Name == "block" || Name == "loop" || Name == "if") {
@@ -406,7 +453,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
                         wasm::WASM_SYMBOL_TYPE_TAG, Sig))
         // catch instruction pushes values whose types are specified in the
         // tag's "params" part
-        Stack.insert(Stack.end(), Sig->Params.begin(), Sig->Params.end());
+        pushTypes(Sig->Params);
       else
         Error = true;
     }
@@ -421,14 +468,14 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   }
 
   if (Name == "return") {
-    return endOfFunction(ErrorLoc);
+    return endOfFunction(ErrorLoc, false);
   }
 
   if (Name == "call_indirect" || Name == "return_call_indirect") {
     // Function value.
     bool Error = popType(ErrorLoc, wasm::ValType::I32);
     Error |= checkSig(ErrorLoc, LastSig);
-    if (Name == "return_call_indirect" && endOfFunction(ErrorLoc))
+    if (Name == "return_call_indirect" && endOfFunction(ErrorLoc, false))
       return true;
     return Error;
   }
@@ -441,7 +488,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
       Error |= checkSig(ErrorLoc, *Sig);
     else
       Error = true;
-    if (Name == "return_call" && endOfFunction(ErrorLoc))
+    if (Name == "return_call" && endOfFunction(ErrorLoc, false))
       return true;
     return Error;
   }
@@ -453,7 +500,7 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
 
   if (Name == "ref.is_null") {
     bool Error = popRefType(ErrorLoc);
-    Stack.push_back(wasm::ValType::I32);
+    pushType(wasm::ValType::I32);
     return Error;
   }
 
@@ -471,22 +518,22 @@ bool WebAssemblyAsmTypeCheck::typeCheck(SMLoc ErrorLoc, const MCInst &Inst,
   auto RegOpc = WebAssembly::getRegisterOpcode(Opc);
   assert(RegOpc != -1 && "Failed to get register version of MC instruction");
   const auto &II = MII.get(RegOpc);
-  bool Error = false;
   // First pop all the uses off the stack and check them.
-  for (unsigned I = II.getNumOperands(); I > II.getNumDefs(); I--) {
-    const auto &Op = II.operands()[I - 1];
-    if (Op.OperandType == MCOI::OPERAND_REGISTER) {
-      auto VT = WebAssembly::regClassToValType(Op.RegClass);
-      Error |= popType(ErrorLoc, VT);
-    }
+  SmallVector<wasm::ValType, 4> PopTypes;
+  for (unsigned I = II.getNumDefs(); I < II.getNumOperands(); I++) {
+    const auto &Op = II.operands()[I];
+    if (Op.OperandType == MCOI::OPERAND_REGISTER)
+      PopTypes.push_back(WebAssembly::regClassToValType(Op.RegClass));
   }
+  bool Error = checkAndPopTypes(ErrorLoc, PopTypes, false);
+  SmallVector<wasm::ValType, 4> PushTypes;
   // Now push all the defs onto the stack.
   for (unsigned I = 0; I < II.getNumDefs(); I++) {
     const auto &Op = II.operands()[I];
     assert(Op.OperandType == MCOI::OPERAND_REGISTER && "Register expected");
-    auto VT = WebAssembly::regClassToValType(Op.RegClass);
-    Stack.push_back(VT);
+    PushTypes.push_back(WebAssembly::regClassToValType(Op.RegClass));
   }
+  pushTypes(PushTypes);
   return Error;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
index 972162d3e02f4..9fd35a26f30e5 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmTypeCheck.h
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCSymbol.h"
+#include <variant>
 
 namespace llvm {
 
@@ -28,7 +29,10 @@ class WebAssemblyAsmTypeCheck final {
   MCAsmParser &Parser;
   const MCInstrInfo &MII;
 
-  SmallVector<wasm::ValType, 8> Stack;
+  struct Ref : public std::monostate {};
+  struct Any : public std::monostate {};
+  using StackType = std::variant<wasm::ValType, Ref, Any>;
+  SmallVector<StackType, 16> Stack;
   SmallVector<SmallVector<wasm::ValType, 4>, 8> BrStack;
   SmallVector<wasm::ValType, 16> LocalTypes;
   SmallVector<wasm::ValType, 4> ReturnTypes;
@@ -36,10 +40,29 @@ class WebAssemblyAsmTypeCheck final {
   bool Unreachable = false;
   bool Is64;
 
+  // If ExactMatch is true, 'Types' will be compared against not only the top of
+  // the value stack but the whole remaining value stack
+  // (TODO: This should be the whole remaining value stack "at the the current
+  // block level", which has not been implemented yet)
+  bool checkTypes(SMLoc ErrorLoc, ArrayRef<wasm::ValType> Types,
+                  bool ExactMatch);
+  bool checkTypes(SMLoc ErrorLoc, ArrayRef<StackType> Types, bool ExactMatch);
+  bool checkAndPopTypes(SMLoc ErrorLoc, ArrayRef<wasm::ValType> Types,
+                        bool ExactMatch);
+  bool checkAndPopTypes(SMLoc ErrorLoc, ArrayRef<StackType> Types,
+                        bool ExactMatch);
+  bool popType(SMLoc ErrorLoc, StackType Type);
+  bool popRefType(SMLoc ErrorLoc);
+  bool popAnyType(SMLoc ErrorLoc);
+  void pushTypes(ArrayRef<wasm::ValType> Types);
+  void pushType(StackType Type) { Stack.push_back(Type); }
+  bool match(StackType TypeA, StackType TypeB);
+  std::string getTypesString(ArrayRef<StackType> Types, size_t StartPos);
+  SmallVector<StackType, 4>
+  valTypeToStackType(ArrayRef<wasm::ValType> ValTypes);
+
   void dumpTypeStack(Twine Msg);
   bool typeError(SMLoc ErrorLoc, const Twine &Msg);
-  bool popType(SMLoc ErrorLoc, std::optional<wasm::ValType> EVT);
-  bool popRefType(SMLoc ErrorLoc);
   bool getLocal(SMLoc ErrorLoc, const MCOperand &LocalOp, wasm::ValType &Type);
   bool checkEnd(SMLoc ErrorLoc, bool PopVals = false);
   bool checkBr(SMLoc ErrorLoc, size_t Level);
@@ -59,7 +82,7 @@ class WebAssemblyAsmTypeCheck final {
   void funcDecl(const wasm::WasmSignature &Sig);
   void localDecl(const SmallVectorImpl<wasm::ValType> &Locals);
   void setLastSig(const wasm::WasmSignature &Sig) { LastSig = Sig; }
-  bool endOfFunction(SMLoc ErrorLoc);
+  bool endOfFunction(SMLoc ErrorLoc, bool ExactMatch);
   bool typeCheck(SMLoc ErrorLoc, const MCInst &Inst, OperandVector &Operands);
 
   void clear() {
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index 307aeb2ea4c6f..91af111db8cda 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -1104,6 +1104,9 @@ def CC_X86 : CallingConv<[
 
 def CSR_NoRegs : CalleeSavedRegs<(add)>;
 
+def CSR_IPRA_32 : CalleeSavedRegs<(add EBP, ESI)>;
+def CSR_IPRA_64 : CalleeSavedRegs<(add RBP, RBX)>;
+
 def CSR_32 : CalleeSavedRegs<(add ESI, EDI, EBX, EBP)>;
 def CSR_64 : CalleeSavedRegs<(add RBX, R12, R13, R14, R15, RBP)>;
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d9eedfdfd53a4..73f7f52846f62 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -35451,11 +35451,11 @@ static MachineInstrBuilder createPHIsForCMOVsInSinkBB(
     if (MIIt->getOperand(3).getImm() == OppCC)
       std::swap(Op1Reg, Op2Reg);
 
-    if (RegRewriteTable.contains(Op1Reg))
-      Op1Reg = RegRewriteTable[Op1Reg].first;
+    if (auto It = RegRewriteTable.find(Op1Reg); It != RegRewriteTable.end())
+      Op1Reg = It->second.first;
 
-    if (RegRewriteTable.contains(Op2Reg))
-      Op2Reg = RegRewriteTable[Op2Reg].second;
+    if (auto It = RegRewriteTable.find(Op2Reg); It != RegRewriteTable.end())
+      Op2Reg = It->second.second;
 
     MIB =
         BuildMI(*SinkMBB, SinkInsertionPoint, MIMD, TII->get(X86::PHI), DestReg)
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 1d8808f4e2b7d..302d50581e1e6 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -410,6 +410,11 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   return CallsEHReturn ? CSR_32EHRet_SaveList : CSR_32_SaveList;
 }
 
+const MCPhysReg *
+X86RegisterInfo::getIPRACSRegs(const MachineFunction *MF) const {
+  return Is64Bit ? CSR_IPRA_64_SaveList : CSR_IPRA_32_SaveList;
+}
+
 const MCPhysReg *X86RegisterInfo::getCalleeSavedRegsViaCopy(
     const MachineFunction *MF) const {
   assert(MF && "Invalid MachineFunction pointer.");
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index 2f73698a4b94d..68ee372f27b14 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -99,6 +99,9 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
   /// callee-save registers on this target.
   const MCPhysReg *
   getCalleeSavedRegs(const MachineFunction* MF) const override;
+  /// getIPRACSRegs - This API can be removed when rbp is safe to optimized out
+  /// when IPRA is on.
+  const MCPhysReg *getIPRACSRegs(const MachineFunction *MF) const override;
   const MCPhysReg *
   getCalleeSavedRegsViaCopy(const MachineFunction *MF) const;
   const uint32_t *getCallPreservedMask(const MachineFunction &MF,
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index f213ac1c8d7d5..96b5c8440e5f9 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -397,13 +397,11 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
     if (!IsOffsetKnown) {
       AliasOffetMap[&I].reset();
     } else {
-      auto Itr = AliasOffetMap.find(&I);
-      if (Itr == AliasOffetMap.end()) {
-        AliasOffetMap[&I] = Offset;
-      } else if (Itr->second && *Itr->second != Offset) {
+      auto [Itr, Inserted] = AliasOffetMap.try_emplace(&I, Offset);
+      if (!Inserted && Itr->second && *Itr->second != Offset) {
         // If we have seen two different possible values for this alias, we set
         // it to empty.
-        AliasOffetMap[&I].reset();
+        Itr->second.reset();
       }
     }
   }
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 1f9b546ed2999..90e8c39e5a90d 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -485,11 +485,36 @@ static bool allCallersPassValidPointerForArgument(
   });
 }
 
+// Try to prove that all Calls to F do not modify the memory pointed to by Arg,
+// using alias analysis local to each caller of F.
+static bool isArgUnmodifiedByAllCalls(Argument *Arg,
+                                      FunctionAnalysisManager &FAM) {
+  for (User *U : Arg->getParent()->users()) {
+
+    // Bail if we find an unexpected (non CallInst) use of the function.
+    auto *Call = dyn_cast<CallInst>(U);
+    if (!Call)
+      return false;
+
+    MemoryLocation Loc =
+        MemoryLocation::getForArgument(Call, Arg->getArgNo(), nullptr);
+
+    AAResults &AAR = FAM.getResult<AAManager>(*Call->getFunction());
+    // Bail as soon as we find a Call where Arg may be modified.
+    if (isModSet(AAR.getModRefInfo(Call, Loc)))
+      return false;
+  }
+
+  // All Users are Calls which do not modify the Arg.
+  return true;
+}
+
 /// Determine that this argument is safe to promote, and find the argument
 /// parts it can be promoted into.
 static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR,
                          unsigned MaxElements, bool IsRecursive,
-                         SmallVectorImpl<OffsetAndArgPart> &ArgPartsVec) {
+                         SmallVectorImpl<OffsetAndArgPart> &ArgPartsVec,
+                         FunctionAnalysisManager &FAM) {
   // Quick exit for unused arguments
   if (Arg->use_empty())
     return true;
@@ -716,10 +741,16 @@ static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR,
     return true;
 
   // Okay, now we know that the argument is only used by load instructions, and
-  // it is safe to unconditionally perform all of them. Use alias analysis to
-  // check to see if the pointer is guaranteed to not be modified from entry of
-  // the function to each of the load instructions.
+  // it is safe to unconditionally perform all of them.
+
+  // If we can determine that no call to the Function modifies the memory region
+  // accessed through Arg, through alias analysis using actual arguments in the
+  // callers, we know that it is guaranteed to be safe to promote the argument.
+  if (isArgUnmodifiedByAllCalls(Arg, FAM))
+    return true;
 
+  // Otherwise, use alias analysis to check if the pointer is guaranteed to not
+  // be modified from entry of the function to each of the load instructions.
   for (LoadInst *Load : Loads) {
     // Check to see if the load is invalidated from the start of the block to
     // the load itself.
@@ -846,7 +877,8 @@ static Function *promoteArguments(Function *F, FunctionAnalysisManager &FAM,
     // If we can promote the pointer to its value.
     SmallVector<OffsetAndArgPart, 4> ArgParts;
 
-    if (findArgParts(PtrArg, DL, AAR, MaxElements, IsRecursive, ArgParts)) {
+    if (findArgParts(PtrArg, DL, AAR, MaxElements, IsRecursive, ArgParts,
+                     FAM)) {
       SmallVector<Type *, 4> Types;
       for (const auto &Pair : ArgParts)
         Types.push_back(Pair.second.Ty);
diff --git a/llvm/lib/Transforms/IPO/CalledValuePropagation.cpp b/llvm/lib/Transforms/IPO/CalledValuePropagation.cpp
index acc10f57c29ac..66ae0706d638c 100644
--- a/llvm/lib/Transforms/IPO/CalledValuePropagation.cpp
+++ b/llvm/lib/Transforms/IPO/CalledValuePropagation.cpp
@@ -169,7 +169,8 @@ class CVPLatticeFunc
   /// just a few kinds of instructions since we're only propagating values that
   /// can be called.
   void ComputeInstructionState(
-      Instruction &I, DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
+      Instruction &I,
+      SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
       SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) override {
     switch (I.getOpcode()) {
     case Instruction::Call:
@@ -238,9 +239,10 @@ class CVPLatticeFunc
 
   /// Handle return instructions. The function's return state is the merge of
   /// the returned value state and the function's return state.
-  void visitReturn(ReturnInst &I,
-                   DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
-                   SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
+  void
+  visitReturn(ReturnInst &I,
+              SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
+              SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     Function *F = I.getParent()->getParent();
     if (F->getReturnType()->isVoidTy())
       return;
@@ -254,9 +256,10 @@ class CVPLatticeFunc
   /// the merge of the argument state with the call sites corresponding actual
   /// argument state. The call site state is the merge of the call site state
   /// with the returned value state of the called function.
-  void visitCallBase(CallBase &CB,
-                     DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
-                     SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
+  void
+  visitCallBase(CallBase &CB,
+                SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
+                SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     Function *F = CB.getCalledFunction();
     auto RegI = CVPLatticeKey(&CB, IPOGrouping::Register);
 
@@ -298,9 +301,10 @@ class CVPLatticeFunc
 
   /// Handle select instructions. The select instruction state is the merge the
   /// true and false value states.
-  void visitSelect(SelectInst &I,
-                   DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
-                   SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
+  void
+  visitSelect(SelectInst &I,
+              SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
+              SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     auto RegI = CVPLatticeKey(&I, IPOGrouping::Register);
     auto RegT = CVPLatticeKey(I.getTrueValue(), IPOGrouping::Register);
     auto RegF = CVPLatticeKey(I.getFalseValue(), IPOGrouping::Register);
@@ -312,7 +316,7 @@ class CVPLatticeFunc
   /// variable, we attempt to track the value. The loaded value state is the
   /// merge of the loaded value state with the global variable state.
   void visitLoad(LoadInst &I,
-                 DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
+                 SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
                  SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     auto RegI = CVPLatticeKey(&I, IPOGrouping::Register);
     if (auto *GV = dyn_cast<GlobalVariable>(I.getPointerOperand())) {
@@ -327,9 +331,10 @@ class CVPLatticeFunc
   /// Handle store instructions. If the pointer operand of the store is a
   /// global variable, we attempt to track the value. The global variable state
   /// is the merge of the stored value state with the global variable state.
-  void visitStore(StoreInst &I,
-                  DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
-                  SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
+  void
+  visitStore(StoreInst &I,
+             SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
+             SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     auto *GV = dyn_cast<GlobalVariable>(I.getPointerOperand());
     if (!GV)
       return;
@@ -342,7 +347,7 @@ class CVPLatticeFunc
   /// Handle all other instructions. All other instructions are marked
   /// overdefined.
   void visitInst(Instruction &I,
-                 DenseMap<CVPLatticeKey, CVPLatticeVal> &ChangedValues,
+                 SmallDenseMap<CVPLatticeKey, CVPLatticeVal, 16> &ChangedValues,
                  SparseSolver<CVPLatticeKey, CVPLatticeVal> &SS) {
     // Simply bail if this instruction has no user.
     if (I.use_empty())
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
index 0742b259c489c..519a4e9314a26 100644
--- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1658,8 +1658,8 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(
                        ".cfi.jumptable", &M);
   ArrayType *JumpTableType =
       ArrayType::get(getJumpTableEntryType(), Functions.size());
-  auto JumpTable =
-      ConstantExpr::getPointerCast(JumpTableFn, JumpTableType->getPointerTo(0));
+  auto JumpTable = ConstantExpr::getPointerCast(
+      JumpTableFn, PointerType::getUnqual(M.getContext()));
 
   lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout);
 
diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
index 576a31f8b86ae..5e7d2c3c713d3 100644
--- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
+++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp
@@ -619,6 +619,16 @@ class CallsiteContextGraph {
     return static_cast<const DerivedCCG *>(this)->getLabel(Func, Call, CloneNo);
   }
 
+  // Create and return a new ContextNode.
+  ContextNode *createNewNode(bool IsAllocation, const FuncTy *F = nullptr,
+                             CallInfo C = CallInfo()) {
+    NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
+    auto *NewNode = NodeOwner.back().get();
+    if (F)
+      NodeToCallingFunc[NewNode] = F;
+    return NewNode;
+  }
+
   /// Helpers to find the node corresponding to the given call or stackid.
   ContextNode *getNodeForInst(const CallInfo &C);
   ContextNode *getNodeForAlloc(const CallInfo &C);
@@ -1082,11 +1092,8 @@ typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
 CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
     CallInfo Call, const FuncTy *F) {
   assert(!getNodeForAlloc(Call));
-  NodeOwner.push_back(
-      std::make_unique<ContextNode>(/*IsAllocation=*/true, Call));
-  ContextNode *AllocNode = NodeOwner.back().get();
+  ContextNode *AllocNode = createNewNode(/*IsAllocation=*/true, F, Call);
   AllocationCallToContextNodeMap[Call] = AllocNode;
-  NodeToCallingFunc[AllocNode] = F;
   // Use LastContextId as a uniq id for MIB allocation nodes.
   AllocNode->OrigStackOrAllocId = LastContextId;
   // Alloc type should be updated as we add in the MIBs. We should assert
@@ -1143,9 +1150,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
     auto StackId = getStackId(*ContextIter);
     ContextNode *StackNode = getNodeForStackId(StackId);
     if (!StackNode) {
-      NodeOwner.push_back(
-          std::make_unique<ContextNode>(/*IsAllocation=*/false));
-      StackNode = NodeOwner.back().get();
+      StackNode = createNewNode(/*IsAllocation=*/false);
       StackEntryIdToContextNodeMap[StackId] = StackNode;
       StackNode->OrigStackOrAllocId = StackId;
     }
@@ -1377,9 +1382,12 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
   // Compute the last node's context ids once, as it is shared by all calls in
   // this entry.
   DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
-  assert(!LastNodeContextIds.empty());
 
-  for (unsigned I = 0; I < Calls.size(); I++) {
+  bool PrevIterCreatedNode = false;
+  bool CreatedNode = false;
+  for (unsigned I = 0; I < Calls.size();
+       I++, PrevIterCreatedNode = CreatedNode) {
+    CreatedNode = false;
     auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
     // Skip any for which we didn't assign any ids, these don't get a node in
     // the graph.
@@ -1391,7 +1399,13 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       if (!CallToMatchingCall.contains(Call))
         continue;
       auto MatchingCall = CallToMatchingCall[Call];
-      assert(NonAllocationCallToContextNodeMap.contains(MatchingCall));
+      if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
+        // This should only happen if we had a prior iteration, and it didn't
+        // create a node because of the below recomputation of context ids
+        // finding none remaining and continuing early.
+        assert(I > 0 && !PrevIterCreatedNode);
+        continue;
+      }
       NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
           Call);
       continue;
@@ -1439,11 +1453,9 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
       continue;
 
     // Create new context node.
-    NodeOwner.push_back(
-        std::make_unique<ContextNode>(/*IsAllocation=*/false, Call));
-    ContextNode *NewNode = NodeOwner.back().get();
-    NodeToCallingFunc[NewNode] = Func;
+    ContextNode *NewNode = createNewNode(/*IsAllocation=*/false, Func, Call);
     NonAllocationCallToContextNodeMap[Call] = NewNode;
+    CreatedNode = true;
     NewNode->AllocTypes = computeAllocType(SavedContextIds);
 
     ContextNode *FirstNode = getNodeForStackId(Ids[0]);
@@ -1548,13 +1560,23 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
     // of length, and within each length, lexicographically by stack id. The
     // latter is so that we can specially handle calls that have identical stack
     // id sequences (either due to cloning or artificially because of the MIB
-    // context pruning).
-    std::stable_sort(Calls.begin(), Calls.end(),
-                     [](const CallContextInfo &A, const CallContextInfo &B) {
-                       return A.StackIds.size() > B.StackIds.size() ||
-                              (A.StackIds.size() == B.StackIds.size() &&
-                               A.StackIds < B.StackIds);
-                     });
+    // context pruning). Those with the same Ids are then sorted by function to
+    // facilitate efficiently mapping them to the same context node.
+    // Because the functions are pointers, to ensure a stable sort first assign
+    // each function pointer to its first index in the Calls array, and then use
+    // that to sort by.
+    DenseMap<const FuncTy *, unsigned> FuncToIndex;
+    for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
+      FuncToIndex.insert({CallCtxInfo.Func, Idx});
+    std::stable_sort(
+        Calls.begin(), Calls.end(),
+        [&FuncToIndex](const CallContextInfo &A, const CallContextInfo &B) {
+          return A.StackIds.size() > B.StackIds.size() ||
+                 (A.StackIds.size() == B.StackIds.size() &&
+                  (A.StackIds < B.StackIds ||
+                   (A.StackIds == B.StackIds &&
+                    FuncToIndex[A.Func] < FuncToIndex[B.Func])));
+        });
 
     // Find the node for the last stack id, which should be the same
     // across all calls recorded for this id, and is the id for this
@@ -1572,18 +1594,26 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
     DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
     assert(!LastNodeContextIds.empty());
 
-    // Map from function to the first call from the below list (with matching
-    // stack ids) found in that function. Note that calls from different
-    // functions can have the same stack ids because this is the list of stack
-    // ids that had (possibly pruned) nodes after building the graph from the
-    // allocation MIBs.
-    DenseMap<const FuncTy *, CallInfo> FuncToCallMap;
+#ifndef NDEBUG
+    // Save the set of functions seen for a particular set of the same stack
+    // ids. This is used to ensure that they have been correctly sorted to be
+    // adjacent in the Calls list, since we rely on that to efficiently place
+    // all such matching calls onto the same context node.
+    DenseSet<const FuncTy *> MatchingIdsFuncSet;
+#endif
 
     for (unsigned I = 0; I < Calls.size(); I++) {
       auto &[Call, Ids, Func, SavedContextIds] = Calls[I];
       assert(SavedContextIds.empty());
       assert(LastId == Ids.back());
 
+#ifndef NDEBUG
+      // If this call has a different set of ids than the last one, clear the
+      // set used to ensure they are sorted properly.
+      if (I > 0 && Ids != Calls[I - 1].StackIds)
+        MatchingIdsFuncSet.clear();
+#endif
+
       // First compute the context ids for this stack id sequence (the
       // intersection of the context ids of the corresponding nodes).
       // Start with the remaining saved ids for the last node.
@@ -1652,23 +1682,38 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
           continue;
       }
 
-      // If the prior call had the same stack ids this map would not be empty.
+#ifndef NDEBUG
+      // If the prior call had the same stack ids this set would not be empty.
       // Check if we already have a call that "matches" because it is located
-      // in the same function.
-      if (FuncToCallMap.contains(Func)) {
-        // Record the matching call found for this call, and skip it. We
-        // will subsequently combine it into the same node.
-        CallToMatchingCall[Call] = FuncToCallMap[Func];
-        continue;
-      }
+      // in the same function. If the Calls list was sorted properly we should
+      // not encounter this situation as all such entries should be adjacent
+      // and processed in bulk further below.
+      assert(!MatchingIdsFuncSet.contains(Func));
+
+      MatchingIdsFuncSet.insert(Func);
+#endif
 
       // Check if the next set of stack ids is the same (since the Calls vector
       // of tuples is sorted by the stack ids we can just look at the next one).
+      // If so, save them in the CallToMatchingCall map so that they get
+      // assigned to the same context node, and skip them.
       bool DuplicateContextIds = false;
-      if (I + 1 < Calls.size()) {
-        auto &CallCtxInfo = Calls[I + 1];
+      for (unsigned J = I + 1; J < Calls.size(); J++) {
+        auto &CallCtxInfo = Calls[J];
         auto &NextIds = CallCtxInfo.StackIds;
-        DuplicateContextIds = Ids == NextIds;
+        if (NextIds != Ids)
+          break;
+        auto *NextFunc = CallCtxInfo.Func;
+        if (NextFunc != Func) {
+          // We have another Call with the same ids but that cannot share this
+          // node, must duplicate ids for it.
+          DuplicateContextIds = true;
+          break;
+        }
+        auto &NextCall = CallCtxInfo.Call;
+        CallToMatchingCall[NextCall] = Call;
+        // Update I so that it gets incremented correctly to skip this call.
+        I = J;
       }
 
       // If we don't have duplicate context ids, then we can assign all the
@@ -1692,14 +1737,7 @@ void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
         set_subtract(LastNodeContextIds, StackSequenceContextIds);
         if (LastNodeContextIds.empty())
           break;
-        // No longer possibly in a sequence of calls with duplicate stack ids,
-        // clear the map.
-        FuncToCallMap.clear();
-      } else
-        // Record the call with its function, so we can locate it the next time
-        // we find a call from this function when processing the calls with the
-        // same stack ids.
-        FuncToCallMap[Func] = Call;
+      }
     }
   }
 
@@ -2128,10 +2166,7 @@ bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
     } else {
       FuncToCallsWithMetadata[Func].push_back({NewCall});
       // Create Node and record node info.
-      NodeOwner.push_back(
-          std::make_unique<ContextNode>(/*IsAllocation=*/false, NewCall));
-      NewNode = NodeOwner.back().get();
-      NodeToCallingFunc[NewNode] = Func;
+      NewNode = createNewNode(/*IsAllocation=*/false, Func, NewCall);
       TailCallToContextNodeMap[NewCall] = NewNode;
       NewNode->AllocTypes = Edge->AllocTypes;
     }
@@ -2704,13 +2739,11 @@ CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
     const std::shared_ptr<ContextEdge> &Edge, EdgeIter *CallerEdgeI,
     DenseSet<uint32_t> ContextIdsToMove) {
   ContextNode *Node = Edge->Callee;
-  NodeOwner.push_back(
-      std::make_unique<ContextNode>(Node->IsAllocation, Node->Call));
-  ContextNode *Clone = NodeOwner.back().get();
+  assert(NodeToCallingFunc.count(Node));
+  ContextNode *Clone =
+      createNewNode(Node->IsAllocation, NodeToCallingFunc[Node], Node->Call);
   Node->addClone(Clone);
   Clone->MatchingCalls = Node->MatchingCalls;
-  assert(NodeToCallingFunc.count(Node));
-  NodeToCallingFunc[Clone] = NodeToCallingFunc[Node];
   moveEdgeToExistingCalleeClone(Edge, Clone, CallerEdgeI, /*NewClone=*/true,
                                 ContextIdsToMove);
   return Clone;
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 542c319b88074..dbc733826944b 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -49,6 +49,13 @@ using namespace llvm;
 STATISTIC(NumInlined, "Number of functions inlined");
 STATISTIC(NumDeleted, "Number of functions deleted because all callers found");
 
+cl::opt<bool> CtxProfPromoteAlwaysInline(
+    "ctx-prof-promote-alwaysinline", cl::init(false), cl::Hidden,
+    cl::desc("If using a contextual profile in this module, and an indirect "
+             "call target is marked as alwaysinline, perform indirect call "
+             "promotion for that target. If multiple targets for an indirect "
+             "call site fit this description, they are all promoted."));
+
 /// Return true if the specified inline history ID
 /// indicates an inline history that includes the specified function.
 static bool inlineHistoryIncludes(
@@ -145,10 +152,11 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
   assert(Calls != nullptr && "Expected an initialized InlineOrder");
 
   // Populate the initial list of calls in this module.
+  SetVector<std::pair<CallBase *, Function *>> ICPCandidates;
   for (Function &F : M) {
     auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);
-    for (Instruction &I : instructions(F))
-      if (auto *CB = dyn_cast<CallBase>(&I))
+    for (Instruction &I : instructions(F)) {
+      if (auto *CB = dyn_cast<CallBase>(&I)) {
         if (Function *Callee = CB->getCalledFunction()) {
           if (!Callee->isDeclaration())
             Calls->push({CB, -1});
@@ -163,7 +171,17 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
                      << setIsVerbose();
             });
           }
+        } else if (CtxProfPromoteAlwaysInline && CtxProf &&
+                   CB->isIndirectCall()) {
+          CtxProfAnalysis::collectIndirectCallPromotionList(*CB, CtxProf,
+                                                            ICPCandidates);
         }
+      }
+    }
+  }
+  for (auto &[CB, Target] : ICPCandidates) {
+    if (auto *DirectCB = promoteCallWithIfThenElse(*CB, *Target, CtxProf))
+      Calls->push({DirectCB, -1});
   }
   if (Calls->empty())
     return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 28da864cad0ff..35664a5c7a2ac 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3536,8 +3536,8 @@ struct AAHeapToSharedFunction : public AAHeapToShared {
           PoisonValue::get(Int8ArrTy), CB->getName() + "_shared", nullptr,
           GlobalValue::NotThreadLocal,
           static_cast<unsigned>(AddressSpace::Shared));
-      auto *NewBuffer =
-          ConstantExpr::getPointerCast(SharedMem, Int8Ty->getPointerTo());
+      auto *NewBuffer = ConstantExpr::getPointerCast(
+          SharedMem, PointerType::getUnqual(M->getContext()));
 
       auto Remark = [&](OptimizationRemark OR) {
         return OR << "Replaced globalized variable with "
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index c66db9285c799..dd31bfa7e65f5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -858,35 +858,16 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Instruction *I,
   }
   case Instruction::SRem: {
     const APInt *Rem;
-    // X % -1 demands all the bits because we don't want to introduce
-    // INT_MIN % -1 (== undef) by accident.
-    if (match(I->getOperand(1), m_APInt(Rem)) && !Rem->isAllOnes()) {
-      APInt RA = Rem->abs();
-      if (RA.isPowerOf2()) {
-        if (DemandedMask.ult(RA))    // srem won't affect demanded bits
-          return I->getOperand(0);
-
-        APInt LowBits = RA - 1;
-        APInt Mask2 = LowBits | APInt::getSignMask(BitWidth);
-        if (SimplifyDemandedBits(I, 0, Mask2, LHSKnown, Depth + 1, Q))
-          return I;
-
-        // The low bits of LHS are unchanged by the srem.
-        Known.Zero = LHSKnown.Zero & LowBits;
-        Known.One = LHSKnown.One & LowBits;
-
-        // If LHS is non-negative or has all low bits zero, then the upper bits
-        // are all zero.
-        if (LHSKnown.isNonNegative() || LowBits.isSubsetOf(LHSKnown.Zero))
-          Known.Zero |= ~LowBits;
+    if (match(I->getOperand(1), m_APInt(Rem)) && Rem->isPowerOf2()) {
+      if (DemandedMask.ult(*Rem)) // srem won't affect demanded bits
+        return I->getOperand(0);
 
-        // If LHS is negative and not all low bits are zero, then the upper bits
-        // are all one.
-        if (LHSKnown.isNegative() && LowBits.intersects(LHSKnown.One))
-          Known.One |= ~LowBits;
-
-        break;
-      }
+      APInt LowBits = *Rem - 1;
+      APInt Mask2 = LowBits | APInt::getSignMask(BitWidth);
+      if (SimplifyDemandedBits(I, 0, Mask2, LHSKnown, Depth + 1, Q))
+        return I;
+      Known = KnownBits::srem(LHSKnown, KnownBits::makeConstant(*Rem));
+      break;
     }
 
     llvm::computeKnownBits(I, Known, Depth, Q);
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 23e9c70b62642..4b1650b93cc1d 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2819,10 +2819,17 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
   if (!BO || !BO->isAssociative())
     return false;
 
-  // TODO: Only hoist ADDs and MULs for now.
+  // TODO: Only hoist ADDs, MULs, FADDs, and FMULs for now.
   Instruction::BinaryOps Opcode = BO->getOpcode();
-  if (Opcode != Instruction::Add && Opcode != Instruction::Mul)
+  switch (Opcode) {
+  case Instruction::Add:
+  case Instruction::Mul:
+  case Instruction::FAdd:
+  case Instruction::FMul:
+    break;
+  default:
     return false;
+  }
 
   bool LVInRHS = L.isLoopInvariant(BO->getOperand(0));
   auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(LVInRHS));
@@ -2857,6 +2864,12 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
     if (auto *I = dyn_cast<Instruction>(Inv))
       I->setHasNoUnsignedWrap(true);
     NewBO->setHasNoUnsignedWrap(true);
+  } else if (Opcode == Instruction::FAdd || Opcode == Instruction::FMul) {
+    // Intersect FMF flags for FADD and FMUL.
+    FastMathFlags Intersect = BO->getFastMathFlags() & BO0->getFastMathFlags();
+    if (auto *I = dyn_cast<Instruction>(Inv))
+      I->setFastMathFlags(Intersect);
+    NewBO->setFastMathFlags(Intersect);
   }
 
   BO->replaceAllUsesWith(NewBO);
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 2f88b19a8d390..e9f212fb482c3 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -2024,7 +2024,7 @@ bool MemCpyOptPass::processImmutArgument(CallBase &CB, unsigned ArgNo) {
     return false;
 
   // 4. The memcpy src must not be modified during the call.
-  if (isModSet(AA->getModRefInfo(&CB, MemoryLocation::getForSource(MDep))))
+  if (isModSet(BAA.getModRefInfo(&CB, MemoryLocation::getForSource(MDep))))
     return false;
 
   LLVM_DEBUG(dbgs() << "MemCpyOptPass: Forwarding memcpy to Immut src:\n"
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 4144c7993b7e4..7bffd4da75a5b 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -503,7 +503,8 @@ static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
 static bool
 DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
   SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
-  DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
+  SmallDenseMap<DebugVariable,
+                std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
       VariableMap;
   for (auto &I : *BB) {
     for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
@@ -584,7 +585,8 @@ static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
     return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
 
   SmallVector<DbgValueInst *, 8> ToBeRemoved;
-  DenseMap<DebugVariable, std::pair<SmallVector<Value *, 4>, DIExpression *>>
+  SmallDenseMap<DebugVariable,
+                std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
       VariableMap;
   for (auto &I : *BB) {
     if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
index 5f872c352429c..4ead5cdcf29c0 100644
--- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
+++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp
@@ -623,36 +623,39 @@ CallBase *llvm::promoteCallWithIfThenElse(CallBase &CB, Function &Callee,
     // All the ctx-es belonging to a function must have the same size counters.
     Ctx.resizeCounters(NewCountersSize);
 
-    // Maybe in this context, the indirect callsite wasn't observed at all
+    // Maybe in this context, the indirect callsite wasn't observed at all. That
+    // would make both direct and indirect BBs cold - which is what we already
+    // have from resising the counters.
     if (!Ctx.hasCallsite(CSIndex))
       return;
     auto &CSData = Ctx.callsite(CSIndex);
-    auto It = CSData.find(CalleeGUID);
 
-    // Maybe we did notice the indirect callsite, but to other targets.
-    if (It == CSData.end())
-      return;
-
-    assert(CalleeGUID == It->second.guid());
-
-    uint32_t DirectCount = It->second.getEntrycount();
-    uint32_t TotalCount = 0;
+    uint64_t TotalCount = 0;
     for (const auto &[_, V] : CSData)
       TotalCount += V.getEntrycount();
+    uint64_t DirectCount = 0;
+    // If we called the direct target, update the DirectCount. If we didn't, we
+    // still want to update the indirect BB (to which the TotalCount goes, in
+    // that case).
+    if (auto It = CSData.find(CalleeGUID); It != CSData.end()) {
+      assert(CalleeGUID == It->second.guid());
+      DirectCount = It->second.getEntrycount();
+      // This direct target needs to be moved to this caller under the
+      // newly-allocated callsite index.
+      assert(Ctx.callsites().count(NewCSID) == 0);
+      Ctx.ingestContext(NewCSID, std::move(It->second));
+      CSData.erase(CalleeGUID);
+    }
+
     assert(TotalCount >= DirectCount);
-    uint32_t IndirectCount = TotalCount - DirectCount;
+    uint64_t IndirectCount = TotalCount - DirectCount;
     // The ICP's effect is as-if the direct BB would have been taken DirectCount
     // times, and the indirect BB, IndirectCount times
     Ctx.counters()[DirectID] = DirectCount;
     Ctx.counters()[IndirectID] = IndirectCount;
 
-    // This particular indirect target needs to be moved to this caller under
-    // the newly-allocated callsite index.
-    assert(Ctx.callsites().count(NewCSID) == 0);
-    Ctx.ingestContext(NewCSID, std::move(It->second));
-    CSData.erase(CalleeGUID);
   };
-  CtxProf.update(ProfileUpdater, &Caller);
+  CtxProf.update(ProfileUpdater, Caller);
   return &DirectCall;
 }
 
diff --git a/llvm/lib/Transforms/Utils/CodeLayout.cpp b/llvm/lib/Transforms/Utils/CodeLayout.cpp
index 95edd27c675d2..baaad8bb48f33 100644
--- a/llvm/lib/Transforms/Utils/CodeLayout.cpp
+++ b/llvm/lib/Transforms/Utils/CodeLayout.cpp
@@ -1427,20 +1427,18 @@ codelayout::computeExtTspLayout(ArrayRef<uint64_t> NodeSizes,
 
 double codelayout::calcExtTspScore(ArrayRef<uint64_t> Order,
                                    ArrayRef<uint64_t> NodeSizes,
-                                   ArrayRef<uint64_t> NodeCounts,
                                    ArrayRef<EdgeCount> EdgeCounts) {
   // Estimate addresses of the blocks in memory.
-  std::vector<uint64_t> Addr(NodeSizes.size(), 0);
-  for (size_t Idx = 1; Idx < Order.size(); Idx++) {
+  SmallVector<uint64_t> Addr(NodeSizes.size(), 0);
+  for (uint64_t Idx = 1; Idx < Order.size(); Idx++)
     Addr[Order[Idx]] = Addr[Order[Idx - 1]] + NodeSizes[Order[Idx - 1]];
-  }
-  std::vector<uint64_t> OutDegree(NodeSizes.size(), 0);
-  for (auto Edge : EdgeCounts)
+  SmallVector<uint64_t> OutDegree(NodeSizes.size(), 0);
+  for (auto &Edge : EdgeCounts)
     ++OutDegree[Edge.src];
 
   // Increase the score for each jump.
   double Score = 0;
-  for (auto Edge : EdgeCounts) {
+  for (auto &Edge : EdgeCounts) {
     bool IsConditional = OutDegree[Edge.src] > 1;
     Score += ::extTSPScore(Addr[Edge.src], NodeSizes[Edge.src], Addr[Edge.dst],
                            Edge.count, IsConditional);
@@ -1449,13 +1447,11 @@ double codelayout::calcExtTspScore(ArrayRef<uint64_t> Order,
 }
 
 double codelayout::calcExtTspScore(ArrayRef<uint64_t> NodeSizes,
-                                   ArrayRef<uint64_t> NodeCounts,
                                    ArrayRef<EdgeCount> EdgeCounts) {
-  std::vector<uint64_t> Order(NodeSizes.size());
-  for (size_t Idx = 0; Idx < NodeSizes.size(); Idx++) {
+  SmallVector<uint64_t> Order(NodeSizes.size());
+  for (uint64_t Idx = 0; Idx < NodeSizes.size(); Idx++)
     Order[Idx] = Idx;
-  }
-  return calcExtTspScore(Order, NodeSizes, NodeCounts, EdgeCounts);
+  return calcExtTspScore(Order, NodeSizes, EdgeCounts);
 }
 
 std::vector<uint64_t> codelayout::computeCacheDirectedLayout(
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 25a01558165da..671b0d0822a5d 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -2375,7 +2375,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
     assert(Deleted);
     (void)Deleted;
   };
-  CtxProf.update(Updater, &Caller);
+  CtxProf.update(Updater, Caller);
   return Ret;
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 8bd3dbf069573..eeff4a9f6a8ba 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -5,7 +5,6 @@ add_llvm_component_library(LLVMVectorize
   LoopVectorize.cpp
   SandboxVectorizer/DependencyGraph.cpp
   SandboxVectorizer/Passes/BottomUpVec.cpp
-  SandboxVectorizer/Region.cpp
   SandboxVectorizer/SandboxVectorizer.cpp
   SLPVectorizer.cpp
   Vectorize.cpp
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index e695902c9d72a..43be72f0f34d4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -79,6 +79,10 @@ static cl::opt<LoopVectorizeHints::ScalableForceKind>
                 "Scalable vectorization is available and favored when the "
                 "cost is inconclusive.")));
 
+static cl::opt<bool> EnableHistogramVectorization(
+    "enable-histogram-loop-vectorization", cl::init(false), cl::Hidden,
+    cl::desc("Enables autovectorization of some loops containing histograms"));
+
 /// Maximum vectorization interleave count.
 static const unsigned MaxInterleaveFactor = 16;
 
@@ -1051,6 +1055,133 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
   return true;
 }
 
+/// Find histogram operations that match high-level code in loops:
+/// \code
+/// buckets[indices[i]]+=step;
+/// \endcode
+///
+/// It matches a pattern starting from \p HSt, which Stores to the 'buckets'
+/// array the computed histogram. It uses a BinOp to sum all counts, storing
+/// them using a loop-variant index Load from the 'indices' input array.
+///
+/// On successful matches it updates the STATISTIC 'HistogramsDetected',
+/// regardless of hardware support. When there is support, it additionally
+/// stores the BinOp/Load pairs in \p HistogramCounts, as well the pointers
+/// used to update histogram in \p HistogramPtrs.
+static bool findHistogram(LoadInst *LI, StoreInst *HSt, Loop *TheLoop,
+                          const PredicatedScalarEvolution &PSE,
+                          SmallVectorImpl<HistogramInfo> &Histograms) {
+
+  // Store value must come from a Binary Operation.
+  Instruction *HPtrInstr = nullptr;
+  BinaryOperator *HBinOp = nullptr;
+  if (!match(HSt, m_Store(m_BinOp(HBinOp), m_Instruction(HPtrInstr))))
+    return false;
+
+  // BinOp must be an Add or a Sub modifying the bucket value by a
+  // loop invariant amount.
+  // FIXME: We assume the loop invariant term is on the RHS.
+  //        Fine for an immediate/constant, but maybe not a generic value?
+  Value *HIncVal = nullptr;
+  if (!match(HBinOp, m_Add(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))) &&
+      !match(HBinOp, m_Sub(m_Load(m_Specific(HPtrInstr)), m_Value(HIncVal))))
+    return false;
+
+  // Make sure the increment value is loop invariant.
+  if (!TheLoop->isLoopInvariant(HIncVal))
+    return false;
+
+  // The address to store is calculated through a GEP Instruction.
+  GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(HPtrInstr);
+  if (!GEP)
+    return false;
+
+  // Restrict address calculation to constant indices except for the last term.
+  Value *HIdx = nullptr;
+  for (Value *Index : GEP->indices()) {
+    if (HIdx)
+      return false;
+    if (!isa<ConstantInt>(Index))
+      HIdx = Index;
+  }
+
+  if (!HIdx)
+    return false;
+
+  // Check that the index is calculated by loading from another array. Ignore
+  // any extensions.
+  // FIXME: Support indices from other sources than a linear load from memory?
+  //        We're currently trying to match an operation looping over an array
+  //        of indices, but there could be additional levels of indirection
+  //        in place, or possibly some additional calculation to form the index
+  //        from the loaded data.
+  Value *VPtrVal;
+  if (!match(HIdx, m_ZExtOrSExtOrSelf(m_Load(m_Value(VPtrVal)))))
+    return false;
+
+  // Make sure the index address varies in this loop, not an outer loop.
+  const auto *AR = dyn_cast<SCEVAddRecExpr>(PSE.getSE()->getSCEV(VPtrVal));
+  if (!AR || AR->getLoop() != TheLoop)
+    return false;
+
+  // Ensure we'll have the same mask by checking that all parts of the histogram
+  // (gather load, update, scatter store) are in the same block.
+  LoadInst *IndexedLoad = cast<LoadInst>(HBinOp->getOperand(0));
+  BasicBlock *LdBB = IndexedLoad->getParent();
+  if (LdBB != HBinOp->getParent() || LdBB != HSt->getParent())
+    return false;
+
+  LLVM_DEBUG(dbgs() << "LV: Found histogram for: " << *HSt << "\n");
+
+  // Store the operations that make up the histogram.
+  Histograms.emplace_back(IndexedLoad, HBinOp, HSt);
+  return true;
+}
+
+bool LoopVectorizationLegality::canVectorizeIndirectUnsafeDependences() {
+  // For now, we only support an IndirectUnsafe dependency that calculates
+  // a histogram
+  if (!EnableHistogramVectorization)
+    return false;
+
+  // Find a single IndirectUnsafe dependency.
+  const MemoryDepChecker::Dependence *IUDep = nullptr;
+  const MemoryDepChecker &DepChecker = LAI->getDepChecker();
+  const auto *Deps = DepChecker.getDependences();
+  // If there were too many dependences, LAA abandons recording them. We can't
+  // proceed safely if we don't know what the dependences are.
+  if (!Deps)
+    return false;
+
+  for (const MemoryDepChecker::Dependence &Dep : *Deps) {
+    // Ignore dependencies that are either known to be safe or can be
+    // checked at runtime.
+    if (MemoryDepChecker::Dependence::isSafeForVectorization(Dep.Type) !=
+        MemoryDepChecker::VectorizationSafetyStatus::Unsafe)
+      continue;
+
+    // We're only interested in IndirectUnsafe dependencies here, where the
+    // address might come from a load from memory. We also only want to handle
+    // one such dependency, at least for now.
+    if (Dep.Type != MemoryDepChecker::Dependence::IndirectUnsafe || IUDep)
+      return false;
+
+    IUDep = &Dep;
+  }
+  if (!IUDep)
+    return false;
+
+  // For now only normal loads and stores are supported.
+  LoadInst *LI = dyn_cast<LoadInst>(IUDep->getSource(DepChecker));
+  StoreInst *SI = dyn_cast<StoreInst>(IUDep->getDestination(DepChecker));
+
+  if (!LI || !SI)
+    return false;
+
+  LLVM_DEBUG(dbgs() << "LV: Checking for a histogram on: " << *SI << "\n");
+  return findHistogram(LI, SI, TheLoop, LAI->getPSE(), Histograms);
+}
+
 bool LoopVectorizationLegality::canVectorizeMemory() {
   LAI = &LAIs.getInfo(*TheLoop);
   const OptimizationRemarkAnalysis *LAR = LAI->getReport();
@@ -1062,7 +1193,7 @@ bool LoopVectorizationLegality::canVectorizeMemory() {
   }
 
   if (!LAI->canVectorizeMemory())
-    return false;
+    return canVectorizeIndirectUnsafeDependences();
 
   if (LAI->hasLoadStoreDependenceInvolvingLoopInvariantAddress()) {
     reportVectorizationFailure("We don't allow storing to uniform addresses",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bd493fb2c1ba1..17c7bc55c4b92 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2325,12 +2325,6 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
                                                VPTransformState &State) {
   assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
 
-  // llvm.experimental.noalias.scope.decl intrinsics must only be duplicated for
-  // the first lane and part.
-  if (isa<NoAliasScopeDeclInst>(Instr))
-    if (!Lane.isFirstLane())
-      return;
-
   // Does this instruction return a value ?
   bool IsVoidRetTy = Instr->getType()->isVoidTy();
 
@@ -6508,8 +6502,33 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
     // We've proven all lanes safe to speculate, fall through.
     [[fallthrough]];
   case Instruction::Add:
+  case Instruction::Sub: {
+    auto Info = Legal->getHistogramInfo(I);
+    if (Info && VF.isVector()) {
+      const HistogramInfo *HGram = Info.value();
+      // Assume that a non-constant update value (or a constant != 1) requires
+      // a multiply, and add that into the cost.
+      InstructionCost MulCost = TTI::TCC_Free;
+      ConstantInt *RHS = dyn_cast<ConstantInt>(I->getOperand(1));
+      if (!RHS || RHS->getZExtValue() != 1)
+        MulCost = TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy);
+
+      // Find the cost of the histogram operation itself.
+      Type *PtrTy = VectorType::get(HGram->Load->getPointerOperandType(), VF);
+      Type *ScalarTy = I->getType();
+      Type *MaskTy = VectorType::get(Type::getInt1Ty(I->getContext()), VF);
+      IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+                                  Type::getVoidTy(I->getContext()),
+                                  {PtrTy, ScalarTy, MaskTy});
+
+      // Add the costs together with the add/sub operation.
+      return TTI.getIntrinsicInstrCost(
+                 ICA, TargetTransformInfo::TCK_RecipThroughput) +
+             MulCost + TTI.getArithmeticInstrCost(I->getOpcode(), VectorTy);
+    }
+    [[fallthrough]];
+  }
   case Instruction::FAdd:
-  case Instruction::Sub:
   case Instruction::FSub:
   case Instruction::Mul:
   case Instruction::FMul:
@@ -8426,6 +8445,30 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
   };
 }
 
+VPHistogramRecipe *
+VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
+                                     ArrayRef<VPValue *> Operands) {
+  // FIXME: Support other operations.
+  unsigned Opcode = HI->Update->getOpcode();
+  assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
+         "Histogram update operation must be an Add or Sub");
+
+  SmallVector<VPValue *, 3> HGramOps;
+  // Bucket address.
+  HGramOps.push_back(Operands[1]);
+  // Increment value.
+  HGramOps.push_back(getVPValueOrAddLiveIn(HI->Update->getOperand(1)));
+
+  // In case of predicated execution (due to tail-folding, or conditional
+  // execution, or both), pass the relevant mask.
+  if (Legal->isMaskRequired(HI->Store))
+    HGramOps.push_back(getBlockInMask(HI->Store->getParent()));
+
+  return new VPHistogramRecipe(Opcode,
+                               make_range(HGramOps.begin(), HGramOps.end()),
+                               HI->Store->getDebugLoc());
+}
+
 void VPRecipeBuilder::fixHeaderPhis() {
   BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
   for (VPHeaderPHIRecipe *R : PhisToFix) {
@@ -8549,6 +8592,10 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
   if (auto *CI = dyn_cast<CallInst>(Instr))
     return tryToWidenCall(CI, Operands, Range);
 
+  if (StoreInst *SI = dyn_cast<StoreInst>(Instr))
+    if (auto HistInfo = Legal->getHistogramInfo(SI))
+      return tryToWidenHistogram(*HistInfo, Operands);
+
   if (isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
     return tryToWidenMemory(Instr, Operands, Range);
 
@@ -9998,6 +10045,19 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     InterleaveLoop = false;
   }
 
+  // If there is a histogram in the loop, do not just interleave without
+  // vectorizing. The order of operations will be incorrect without the
+  // histogram intrinsics, which are only used for recipes with VF > 1.
+  if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
+    LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
+                      << "to histogram operations.\n");
+    IntDiagMsg = std::make_pair(
+        "HistogramPreventsScalarInterleaving",
+        "Unable to interleave without vectorization due to constraints on "
+        "the order of histogram operations");
+    InterleaveLoop = false;
+  }
+
   // Override IC if user provided an interleave count.
   IC = UserIC > 0 ? UserIC : IC;
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c6f35c700b2e0..e45fcb2b5c790 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -260,6 +260,20 @@ static FixedVectorType *getWidenedType(Type *ScalarTy, unsigned VF) {
                               VF * getNumElements(ScalarTy));
 }
 
+/// Returns the number of elements of the given type \p Ty, not less than \p Sz,
+/// which forms type, which splits by \p TTI into whole vector types during
+/// legalization.
+static unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI,
+                                              Type *Ty, unsigned Sz) {
+  if (!isValidElementType(Ty))
+    return bit_ceil(Sz);
+  // Find the number of elements, which forms full vectors.
+  const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
+  if (NumParts == 0 || NumParts >= Sz)
+    return bit_ceil(Sz);
+  return bit_ceil(divideCeil(Sz, NumParts)) * NumParts;
+}
+
 static void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements,
                                                    SmallVectorImpl<int> &Mask) {
   // The ShuffleBuilder implementation use shufflevector to splat an "element".
@@ -394,7 +408,7 @@ static bool isVectorLikeInstWithConstOps(Value *V) {
 /// total number of elements \p Size and number of registers (parts) \p
 /// NumParts.
 static unsigned getPartNumElems(unsigned Size, unsigned NumParts) {
-  return PowerOf2Ceil(divideCeil(Size, NumParts));
+  return std::min<unsigned>(Size, bit_ceil(divideCeil(Size, NumParts)));
 }
 
 /// Returns correct remaining number of elements, considering total amount \p
@@ -1222,6 +1236,22 @@ static bool doesNotNeedToSchedule(ArrayRef<Value *> VL) {
          (all_of(VL, isUsedOutsideBlock) || all_of(VL, areAllOperandsNonInsts));
 }
 
+/// Returns true if widened type of \p Ty elements with size \p Sz represents
+/// full vector type, i.e. adding extra element results in extra parts upon type
+/// legalization.
+static bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
+                                     unsigned Sz) {
+  if (Sz <= 1)
+    return false;
+  if (!isValidElementType(Ty) && !isa<FixedVectorType>(Ty))
+    return false;
+  if (has_single_bit(Sz))
+    return true;
+  const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
+  return NumParts > 0 && NumParts < Sz && has_single_bit(Sz / NumParts) &&
+         Sz % NumParts == 0;
+}
+
 namespace slpvectorizer {
 
 /// Bottom Up SLP Vectorizer.
@@ -3311,6 +3341,15 @@ class BoUpSLP {
     /// Return true if this is a non-power-of-2 node.
     bool isNonPowOf2Vec() const {
       bool IsNonPowerOf2 = !has_single_bit(Scalars.size());
+      return IsNonPowerOf2;
+    }
+
+    /// Return true if this is a node, which tries to vectorize number of
+    /// elements, forming whole vectors.
+    bool
+    hasNonWholeRegisterOrNonPowerOf2Vec(const TargetTransformInfo &TTI) const {
+      bool IsNonPowerOf2 = !hasFullVectorsOrPowerOf2(
+          TTI, getValueType(Scalars.front()), Scalars.size());
       assert((!IsNonPowerOf2 || ReuseShuffleIndices.empty()) &&
              "Reshuffling not supported with non-power-of-2 vectors yet.");
       return IsNonPowerOf2;
@@ -3430,8 +3469,10 @@ class BoUpSLP {
     Last->State = EntryState;
     // FIXME: Remove once support for ReuseShuffleIndices has been implemented
     // for non-power-of-two vectors.
-    assert((has_single_bit(VL.size()) || ReuseShuffleIndices.empty()) &&
-           "Reshuffling scalars not yet supported for nodes with padding");
+    assert(
+        (hasFullVectorsOrPowerOf2(*TTI, getValueType(VL.front()), VL.size()) ||
+         ReuseShuffleIndices.empty()) &&
+        "Reshuffling scalars not yet supported for nodes with padding");
     Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
                                      ReuseShuffleIndices.end());
     if (ReorderIndices.empty()) {
@@ -4412,7 +4453,10 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
     return std::nullopt;
   auto *VecTy = getWidenedType(ScalarTy, NumScalars);
   int NumParts = TTI->getNumberOfParts(VecTy);
-  if (NumParts == 0 || NumParts >= NumScalars)
+  if (NumParts == 0 || NumParts >= NumScalars ||
+      VecTy->getNumElements() % NumParts != 0 ||
+      !hasFullVectorsOrPowerOf2(*TTI, VecTy->getElementType(),
+                                VecTy->getNumElements() / NumParts))
     NumParts = 1;
   SmallVector<int> ExtractMask;
   SmallVector<int> Mask;
@@ -5269,7 +5313,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
   // node.
   if (!TE.ReuseShuffleIndices.empty()) {
     // FIXME: Support ReuseShuffleIndices for non-power-of-two vectors.
-    assert(!TE.isNonPowOf2Vec() &&
+    assert(!TE.hasNonWholeRegisterOrNonPowerOf2Vec(*TTI) &&
            "Reshuffling scalars not yet supported for nodes with padding");
 
     if (isSplat(TE.Scalars))
@@ -5331,8 +5375,11 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
             UndefCnt > Sz / 2)
           return std::nullopt;
         UsedVals.set(Val);
-        for (unsigned K = 0; K < NumParts; ++K)
-          ResOrder[Val + Sz * K] = I + K;
+        for (unsigned K = 0; K < NumParts; ++K) {
+          unsigned Idx = Val + Sz * K;
+          if (Idx < VF)
+            ResOrder[Idx] = I + K;
+        }
       }
       return std::move(ResOrder);
     }
@@ -5509,7 +5556,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
     }
     // FIXME: Remove the non-power-of-two check once findReusedOrderedScalars
     // has been auditted for correctness with non-power-of-two vectors.
-    if (!TE.isNonPowOf2Vec())
+    if (!TE.hasNonWholeRegisterOrNonPowerOf2Vec(*TTI))
       if (std::optional<OrdersType> CurrentOrder = findReusedOrderedScalars(TE))
         return CurrentOrder;
   }
@@ -5662,8 +5709,8 @@ void BoUpSLP::reorderTopToBottom() {
   });
 
   // Reorder the graph nodes according to their vectorization factor.
-  for (unsigned VF = VectorizableTree.front()->getVectorFactor(); VF > 1;
-       VF = bit_ceil(VF) / 2) {
+  for (unsigned VF = VectorizableTree.front()->getVectorFactor();
+       !VFToOrderedEntries.empty() && VF > 1; VF -= 2 - (VF & 1U)) {
     auto It = VFToOrderedEntries.find(VF);
     if (It == VFToOrderedEntries.end())
       continue;
@@ -5671,6 +5718,9 @@ void BoUpSLP::reorderTopToBottom() {
     // used order and reorder scalar elements in the nodes according to this
     // mostly used order.
     ArrayRef<TreeEntry *> OrderedEntries = It->second.getArrayRef();
+    // Delete VF entry upon exit.
+    auto Cleanup = make_scope_exit([&]() { VFToOrderedEntries.erase(It); });
+
     // All operands are reordered and used only in this node - propagate the
     // most used order to the user node.
     MapVector<OrdersType, unsigned,
@@ -6413,7 +6463,10 @@ static void gatherPossiblyVectorizableLoads(
   if (NumScalars > 1) {
     auto *VecTy = getWidenedType(ScalarTy, NumScalars);
     NumParts = TTI.getNumberOfParts(VecTy);
-    if (NumParts == 0 || NumParts >= NumScalars)
+    if (NumParts == 0 || NumParts >= NumScalars ||
+        VecTy->getNumElements() % NumParts != 0 ||
+        !hasFullVectorsOrPowerOf2(TTI, VecTy->getElementType(),
+                                  VecTy->getNumElements() / NumParts))
       NumParts = 1;
   }
   unsigned VF = PowerOf2Ceil(NumScalars / NumParts);
@@ -7515,7 +7568,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
   auto TryToFindDuplicates = [&](const InstructionsState &S,
                                  bool DoNotFail = false) {
     // Check that every instruction appears once in this bundle.
-    DenseMap<Value *, unsigned> UniquePositions(VL.size());
+    SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
     for (Value *V : VL) {
       if (isConstant(V)) {
         ReuseShuffleIndices.emplace_back(
@@ -7529,25 +7582,26 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         UniqueValues.emplace_back(V);
     }
     size_t NumUniqueScalarValues = UniqueValues.size();
-    if (NumUniqueScalarValues == VL.size()) {
+    bool IsFullVectors = hasFullVectorsOrPowerOf2(
+        *TTI, UniqueValues.front()->getType(), NumUniqueScalarValues);
+    if (NumUniqueScalarValues == VL.size() &&
+        (VectorizeNonPowerOf2 || IsFullVectors)) {
       ReuseShuffleIndices.clear();
     } else {
       // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
-      if ((UserTreeIdx.UserTE && UserTreeIdx.UserTE->isNonPowOf2Vec()) ||
-          !llvm::has_single_bit(VL.size())) {
+      if ((UserTreeIdx.UserTE &&
+           UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(*TTI)) ||
+          !has_single_bit(VL.size())) {
         LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
                              "for nodes with padding.\n");
         newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
         return false;
       }
       LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
-      if (NumUniqueScalarValues <= 1 ||
-          (UniquePositions.size() == 1 && all_of(UniqueValues,
-                                                 [](Value *V) {
-                                                   return isa<UndefValue>(V) ||
-                                                          !isConstant(V);
-                                                 })) ||
-          !llvm::has_single_bit<uint32_t>(NumUniqueScalarValues)) {
+      if (NumUniqueScalarValues <= 1 || !IsFullVectors ||
+          (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
+             return isa<UndefValue>(V) || !isConstant(V);
+           }))) {
         if (DoNotFail && UniquePositions.size() > 1 &&
             NumUniqueScalarValues > 1 && S.MainOp->isSafeToRemove() &&
             all_of(UniqueValues, [=](Value *V) {
@@ -7555,7 +7609,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                      areAllUsersVectorized(cast<Instruction>(V),
                                            UserIgnoreList);
             })) {
-          unsigned PWSz = PowerOf2Ceil(UniqueValues.size());
+          // Find the number of elements, which forms full vectors.
+          unsigned PWSz = getFullVectorNumberOfElements(
+              *TTI, UniqueValues.front()->getType(), UniqueValues.size());
           if (PWSz == VL.size()) {
             ReuseShuffleIndices.clear();
           } else {
@@ -9793,9 +9849,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
       return nullptr;
     Value *VecBase = nullptr;
     ArrayRef<Value *> VL = E->Scalars;
-    // If the resulting type is scalarized, do not adjust the cost.
-    if (NumParts == VL.size())
-      return nullptr;
     // Check if it can be considered reused if same extractelements were
     // vectorized already.
     bool PrevNodeFound = any_of(
@@ -9911,7 +9964,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     assert(!CommonMask.empty() && "Expected non-empty common mask.");
     auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
     unsigned NumParts = TTI.getNumberOfParts(MaskVecTy);
-    if (NumParts == 0 || NumParts >= Mask.size())
+    if (NumParts == 0 || NumParts >= Mask.size() ||
+        MaskVecTy->getNumElements() % NumParts != 0 ||
+        !hasFullVectorsOrPowerOf2(TTI, MaskVecTy->getElementType(),
+                                  MaskVecTy->getNumElements() / NumParts))
       NumParts = 1;
     unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
     const auto *It =
@@ -9928,7 +9984,10 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
     assert(!CommonMask.empty() && "Expected non-empty common mask.");
     auto *MaskVecTy = getWidenedType(ScalarTy, Mask.size());
     unsigned NumParts = TTI.getNumberOfParts(MaskVecTy);
-    if (NumParts == 0 || NumParts >= Mask.size())
+    if (NumParts == 0 || NumParts >= Mask.size() ||
+        MaskVecTy->getNumElements() % NumParts != 0 ||
+        !hasFullVectorsOrPowerOf2(TTI, MaskVecTy->getElementType(),
+                                  MaskVecTy->getNumElements() / NumParts))
       NumParts = 1;
     unsigned SliceSize = getPartNumElems(Mask.size(), NumParts);
     const auto *It =
@@ -10289,13 +10348,16 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
         InstructionCost VecCost = VectorCost(CommonCost);
         // Check if the current node must be resized, if the parent node is not
         // resized.
-        if (!UnaryInstruction::isCast(E->getOpcode()) && E->Idx != 0 &&
+        if (It != MinBWs.end() && !UnaryInstruction::isCast(E->getOpcode()) &&
+            E->Idx != 0 &&
             (E->getOpcode() != Instruction::Load ||
              !E->UserTreeIndices.empty())) {
-          const EdgeInfo &EI = E->UserTreeIndices.front();
-          if ((EI.UserTE->getOpcode() != Instruction::Select ||
-               EI.EdgeIdx != 0) &&
-              It != MinBWs.end()) {
+          const EdgeInfo &EI =
+              *find_if(E->UserTreeIndices, [](const EdgeInfo &EI) {
+                return !EI.UserTE->isGather() || EI.EdgeIdx != UINT_MAX;
+              });
+          if (EI.UserTE->getOpcode() != Instruction::Select ||
+              EI.EdgeIdx != 0) {
             auto UserBWIt = MinBWs.find(EI.UserTE);
             Type *UserScalarTy =
                 EI.UserTE->getOperand(EI.EdgeIdx).front()->getType();
@@ -10306,8 +10368,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
               unsigned BWSz = DL->getTypeSizeInBits(ScalarTy);
               unsigned SrcBWSz = DL->getTypeSizeInBits(UserScalarTy);
               unsigned VecOpcode;
-              auto *UserVecTy =
-                  getWidenedType(UserScalarTy, E->getVectorFactor());
+              auto *UserVecTy = getWidenedType(UserScalarTy, E->Scalars.size());
               if (BWSz > SrcBWSz)
                 VecOpcode = Instruction::Trunc;
               else
@@ -10450,7 +10511,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       InsertMask[Idx] = I + 1;
     }
     unsigned VecScalarsSz = PowerOf2Ceil(NumElts);
-    if (NumOfParts > 0)
+    if (NumOfParts > 0 && NumOfParts < NumElts)
       VecScalarsSz = PowerOf2Ceil((NumElts + NumOfParts - 1) / NumOfParts);
     unsigned VecSz = (1 + OffsetEnd / VecScalarsSz - OffsetBeg / VecScalarsSz) *
                      VecScalarsSz;
@@ -13579,7 +13640,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
   Type *OrigScalarTy = GatheredScalars.front()->getType();
   auto *VecTy = getWidenedType(ScalarTy, GatheredScalars.size());
   unsigned NumParts = TTI->getNumberOfParts(VecTy);
-  if (NumParts == 0 || NumParts >= GatheredScalars.size())
+  if (NumParts == 0 || NumParts >= GatheredScalars.size() ||
+      VecTy->getNumElements() % NumParts != 0 ||
+      !hasFullVectorsOrPowerOf2(*TTI, VecTy->getElementType(),
+                                VecTy->getNumElements() / NumParts))
     NumParts = 1;
   if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
     // Check for gathered extracts.
@@ -17785,7 +17849,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
     for (unsigned I = NextInst; I < MaxInst; ++I) {
       unsigned ActualVF = std::min(MaxInst - I, VF);
 
-      if (!has_single_bit(ActualVF))
+      if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF))
         continue;
 
       if (MaxVFOnly && ActualVF < MaxVF)
@@ -18501,7 +18565,8 @@ class HorizontalReduction {
       for (Value *V : Candidates)
         TrackedVals.try_emplace(V, V);
 
-    auto At = [](MapVector<Value *, unsigned> &MV, Value *V) -> unsigned & {
+    auto At = [](SmallMapVector<Value *, unsigned, 16> &MV,
+                 Value *V) -> unsigned & {
       auto *It = MV.find(V);
       assert(It != MV.end() && "Unable to find given key.");
       return It->second;
@@ -18588,7 +18653,7 @@ class HorizontalReduction {
                                     RdxKind != RecurKind::FMul &&
                                     RdxKind != RecurKind::FMulAdd;
       // Gather same values.
-      MapVector<Value *, unsigned> SameValuesCounter;
+      SmallMapVector<Value *, unsigned, 16> SameValuesCounter;
       if (IsSupportedHorRdxIdentityOp)
         for (Value *V : Candidates) {
           Value *OrigV = TrackedToOrig.at(V);
@@ -19207,10 +19272,10 @@ class HorizontalReduction {
 
   /// Emits actual operation for the scalar identity values, found during
   /// horizontal reduction analysis.
-  Value *emitReusedOps(Value *VectorizedValue, IRBuilderBase &Builder,
-                       BoUpSLP &R,
-                       const MapVector<Value *, unsigned> &SameValuesCounter,
-                       const DenseMap<Value *, Value *> &TrackedToOrig) {
+  Value *
+  emitReusedOps(Value *VectorizedValue, IRBuilderBase &Builder, BoUpSLP &R,
+                const SmallMapVector<Value *, unsigned, 16> &SameValuesCounter,
+                const DenseMap<Value *, Value *> &TrackedToOrig) {
     assert(IsSupportedHorRdxIdentityOp &&
            "The optimization of matched scalar identity horizontal reductions "
            "must be supported.");
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
index 139e581ce03d9..67b56451c7b59 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.cpp
@@ -31,11 +31,11 @@ void DGNode::dump() const {
 }
 #endif // NDEBUG
 
-InstrInterval DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
+Interval<Instruction> DependencyGraph::extend(ArrayRef<Instruction *> Instrs) {
   if (Instrs.empty())
     return {};
   // TODO: For now create a chain of dependencies.
-  InstrInterval Interval(Instrs);
+  Interval<Instruction> Interval(Instrs);
   auto *TopI = Interval.top();
   auto *BotI = Interval.bottom();
   DGNode *LastN = getOrCreateNode(TopI);
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
index 0c44d05f0474d..7ebbcabb004df 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/SandboxIR/Instruction.h"
 
 using namespace llvm::sandboxir;
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
index 161d300e6e9f2..80afcb499a2c2 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.cpp
@@ -8,8 +8,8 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/SandboxVectorizer.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/SandboxIR/Constant.h"
 #include "llvm/SandboxIR/PassManager.h"
-#include "llvm/SandboxIR/SandboxIR.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Passes/BottomUpVec.h"
 
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 25b8bf3e089e5..02ef6f7b6fb98 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -21,6 +21,7 @@ namespace llvm {
 class LoopVectorizationLegality;
 class LoopVectorizationCostModel;
 class TargetLibraryInfo;
+struct HistogramInfo;
 
 /// Helper class to create VPRecipies from IR instructions.
 class VPRecipeBuilder {
@@ -103,6 +104,13 @@ class VPRecipeBuilder {
   VPWidenRecipe *tryToWiden(Instruction *I, ArrayRef<VPValue *> Operands,
                             VPBasicBlock *VPBB);
 
+  /// Makes Histogram count operations safe for vectorization, by emitting a
+  /// llvm.experimental.vector.histogram.add intrinsic in place of the
+  /// Load + Add|Sub + Store operations that perform the histogram in the
+  /// original scalar loop.
+  VPHistogramRecipe *tryToWidenHistogram(const HistogramInfo *HI,
+                                         ArrayRef<VPValue *> Operands);
+
 public:
   VPRecipeBuilder(VPlan &Plan, Loop *OrigLoop, const TargetLibraryInfo *TLI,
                   LoopVectorizationLegality *Legal,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bbcfaf9e19cd0..c4567362eaffc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -907,6 +907,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue {
     case VPRecipeBase::VPWidenLoadSC:
     case VPRecipeBase::VPWidenStoreEVLSC:
     case VPRecipeBase::VPWidenStoreSC:
+    case VPRecipeBase::VPHistogramSC:
       // TODO: Widened stores don't define a value, but widened loads do. Split
       // the recipes to be able to make widened loads VPSingleDefRecipes.
       return false;
@@ -1664,6 +1665,51 @@ class VPWidenCallRecipe : public VPSingleDefRecipe {
 #endif
 };
 
+/// A recipe representing a sequence of load -> update -> store as part of
+/// a histogram operation. This means there may be aliasing between vector
+/// lanes, which is handled by the llvm.experimental.vector.histogram family
+/// of intrinsics. The only update operations currently supported are
+/// 'add' and 'sub' where the other term is loop-invariant.
+class VPHistogramRecipe : public VPRecipeBase {
+  /// Opcode of the update operation, currently either add or sub.
+  unsigned Opcode;
+
+public:
+  template <typename IterT>
+  VPHistogramRecipe(unsigned Opcode, iterator_range<IterT> Operands,
+                    DebugLoc DL = {})
+      : VPRecipeBase(VPDef::VPHistogramSC, Operands, DL), Opcode(Opcode) {}
+
+  ~VPHistogramRecipe() override = default;
+
+  VPHistogramRecipe *clone() override {
+    return new VPHistogramRecipe(Opcode, operands(), getDebugLoc());
+  }
+
+  VP_CLASSOF_IMPL(VPDef::VPHistogramSC);
+
+  /// Produce a vectorized histogram operation.
+  void execute(VPTransformState &State) override;
+
+  /// Return the cost of this VPHistogramRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
+  unsigned getOpcode() const { return Opcode; }
+
+  /// Return the mask operand if one was provided, or a null pointer if all
+  /// lanes should be executed unconditionally.
+  VPValue *getMask() const {
+    return getNumOperands() == 3 ? getOperand(2) : nullptr;
+  }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// Print the recipe
+  void print(raw_ostream &O, const Twine &Indent,
+             VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
 /// A recipe for widening select instructions.
 struct VPWidenSelectRecipe : public VPSingleDefRecipe {
   template <typename IterT>
@@ -2709,6 +2755,10 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
   /// Generate the wide load or gather.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenLoadEVLRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
@@ -2787,6 +2837,10 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
   /// Generate the wide store or scatter.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenStoreEVLRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   /// Print the recipe.
   void print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 4ddbd0d5fafb8..ed0bb13d9425f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -26,7 +26,7 @@ namespace llvm {
 namespace VPlanPatternMatch {
 
 template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
-  return const_cast<Pattern &>(P).match(V);
+  return P.match(V);
 }
 
 template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
@@ -35,7 +35,7 @@ template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
 }
 
 template <typename Class> struct class_match {
-  template <typename ITy> bool match(ITy *V) { return isa<Class>(V); }
+  template <typename ITy> bool match(ITy *V) const { return isa<Class>(V); }
 };
 
 /// Match an arbitrary VPValue and ignore it.
@@ -46,7 +46,7 @@ template <typename Class> struct bind_ty {
 
   bind_ty(Class *&V) : VR(V) {}
 
-  template <typename ITy> bool match(ITy *V) {
+  template <typename ITy> bool match(ITy *V) const {
     if (auto *CV = dyn_cast<Class>(V)) {
       VR = CV;
       return true;
@@ -63,7 +63,7 @@ template <unsigned BitWidth = 0> struct specific_intval {
 
   specific_intval(APInt V) : Val(std::move(V)) {}
 
-  bool match(VPValue *VPV) {
+  bool match(VPValue *VPV) const {
     if (!VPV->isLiveIn())
       return false;
     Value *V = VPV->getLiveInIRValue();
@@ -94,7 +94,7 @@ template <typename LTy, typename RTy> struct match_combine_or {
 
   match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
 
-  template <typename ITy> bool match(ITy *V) {
+  template <typename ITy> bool match(ITy *V) const {
     if (L.match(V))
       return true;
     if (R.match(V))
@@ -139,16 +139,16 @@ struct UnaryRecipe_match {
 
   UnaryRecipe_match(Op0_t Op0) : Op0(Op0) {}
 
-  bool match(const VPValue *V) {
+  bool match(const VPValue *V) const {
     auto *DefR = V->getDefiningRecipe();
     return DefR && match(DefR);
   }
 
-  bool match(const VPSingleDefRecipe *R) {
+  bool match(const VPSingleDefRecipe *R) const {
     return match(static_cast<const VPRecipeBase *>(R));
   }
 
-  bool match(const VPRecipeBase *R) {
+  bool match(const VPRecipeBase *R) const {
     if (!detail::MatchRecipeAndOpcode<Opcode, RecipeTys...>::match(R))
       return false;
     assert(R->getNumOperands() == 1 &&
@@ -174,16 +174,16 @@ struct BinaryRecipe_match {
 
   BinaryRecipe_match(Op0_t Op0, Op1_t Op1) : Op0(Op0), Op1(Op1) {}
 
-  bool match(const VPValue *V) {
+  bool match(const VPValue *V) const {
     auto *DefR = V->getDefiningRecipe();
     return DefR && match(DefR);
   }
 
-  bool match(const VPSingleDefRecipe *R) {
+  bool match(const VPSingleDefRecipe *R) const {
     return match(static_cast<const VPRecipeBase *>(R));
   }
 
-  bool match(const VPRecipeBase *R) {
+  bool match(const VPRecipeBase *R) const {
     if (!detail::MatchRecipeAndOpcode<Opcode, RecipeTys...>::match(R))
       return false;
     assert(R->getNumOperands() == 2 &&
@@ -314,12 +314,14 @@ m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
 }
 
 struct VPCanonicalIVPHI_match {
-  bool match(const VPValue *V) {
+  bool match(const VPValue *V) const {
     auto *DefR = V->getDefiningRecipe();
     return DefR && match(DefR);
   }
 
-  bool match(const VPRecipeBase *R) { return isa<VPCanonicalIVPHIRecipe>(R); }
+  bool match(const VPRecipeBase *R) const {
+    return isa<VPCanonicalIVPHIRecipe>(R);
+  }
 };
 
 inline VPCanonicalIVPHI_match m_CanonicalIV() {
@@ -332,12 +334,12 @@ template <typename Op0_t, typename Op1_t> struct VPScalarIVSteps_match {
 
   VPScalarIVSteps_match(Op0_t Op0, Op1_t Op1) : Op0(Op0), Op1(Op1) {}
 
-  bool match(const VPValue *V) {
+  bool match(const VPValue *V) const {
     auto *DefR = V->getDefiningRecipe();
     return DefR && match(DefR);
   }
 
-  bool match(const VPRecipeBase *R) {
+  bool match(const VPRecipeBase *R) const {
     if (!isa<VPScalarIVStepsRecipe>(R))
       return false;
     assert(R->getNumOperands() == 2 &&
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index dacba152611c1..7590863853295 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -22,6 +22,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/IR/VectorBuilder.h"
@@ -48,6 +49,24 @@ extern cl::opt<unsigned> ForceTargetInstructionCost;
 
 bool VPRecipeBase::mayWriteToMemory() const {
   switch (getVPDefID()) {
+  case VPInstructionSC:
+    if (Instruction::isBinaryOp(cast<VPInstruction>(this)->getOpcode()))
+      return false;
+    switch (cast<VPInstruction>(this)->getOpcode()) {
+    case Instruction::Or:
+    case Instruction::ICmp:
+    case Instruction::Select:
+    case VPInstruction::Not:
+    case VPInstruction::CalculateTripCountMinusVF:
+    case VPInstruction::CanonicalIVIncrementForPart:
+    case VPInstruction::ExtractFromEnd:
+    case VPInstruction::FirstOrderRecurrenceSplice:
+    case VPInstruction::LogicalAnd:
+    case VPInstruction::PtrAdd:
+      return false;
+    default:
+      return true;
+    }
   case VPInterleaveSC:
     return cast<VPInterleaveRecipe>(this)->getNumStoreOperands() > 0;
   case VPWidenStoreEVLSC:
@@ -137,21 +156,7 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPScalarCastSC:
     return false;
   case VPInstructionSC:
-    switch (cast<VPInstruction>(this)->getOpcode()) {
-    case Instruction::Or:
-    case Instruction::ICmp:
-    case Instruction::Select:
-    case VPInstruction::Not:
-    case VPInstruction::CalculateTripCountMinusVF:
-    case VPInstruction::CanonicalIVIncrementForPart:
-    case VPInstruction::ExtractFromEnd:
-    case VPInstruction::FirstOrderRecurrenceSplice:
-    case VPInstruction::LogicalAnd:
-    case VPInstruction::PtrAdd:
-      return false;
-    default:
-      return true;
-    }
+    return mayWriteToMemory();
   case VPWidenCallSC: {
     Function *Fn = cast<VPWidenCallRecipe>(this)->getCalledScalarFunction();
     return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn();
@@ -1021,6 +1026,94 @@ void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent,
     O << ")";
   }
 }
+#endif
+
+void VPHistogramRecipe::execute(VPTransformState &State) {
+  State.setDebugLocFrom(getDebugLoc());
+  IRBuilderBase &Builder = State.Builder;
+
+  Value *Address = State.get(getOperand(0));
+  Value *IncAmt = State.get(getOperand(1), /*IsScalar=*/true);
+  VectorType *VTy = cast<VectorType>(Address->getType());
+
+  // The histogram intrinsic requires a mask even if the recipe doesn't;
+  // if the mask operand was omitted then all lanes should be executed and
+  // we just need to synthesize an all-true mask.
+  Value *Mask = nullptr;
+  if (VPValue *VPMask = getMask())
+    Mask = State.get(VPMask);
+  else
+    Mask =
+        Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));
+
+  // If this is a subtract, we want to invert the increment amount. We may
+  // add a separate intrinsic in future, but for now we'll try this.
+  if (Opcode == Instruction::Sub)
+    IncAmt = Builder.CreateNeg(IncAmt);
+  else
+    assert(Opcode == Instruction::Add && "only add or sub supported for now");
+
+  State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
+                                {VTy, IncAmt->getType()},
+                                {Address, IncAmt, Mask});
+}
+
+InstructionCost VPHistogramRecipe::computeCost(ElementCount VF,
+                                               VPCostContext &Ctx) const {
+  // FIXME: Take the gather and scatter into account as well. For now we're
+  //        generating the same cost as the fallback path, but we'll likely
+  //        need to create a new TTI method for determining the cost, including
+  //        whether we can use base + vec-of-smaller-indices or just
+  //        vec-of-pointers.
+  assert(VF.isVector() && "Invalid VF for histogram cost");
+  Type *AddressTy = Ctx.Types.inferScalarType(getOperand(0));
+  VPValue *IncAmt = getOperand(1);
+  Type *IncTy = Ctx.Types.inferScalarType(IncAmt);
+  VectorType *VTy = VectorType::get(IncTy, VF);
+
+  // Assume that a non-constant update value (or a constant != 1) requires
+  // a multiply, and add that into the cost.
+  InstructionCost MulCost =
+      Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy);
+  if (IncAmt->isLiveIn()) {
+    ConstantInt *CI = dyn_cast<ConstantInt>(IncAmt->getLiveInIRValue());
+
+    if (CI && CI->getZExtValue() == 1)
+      MulCost = TTI::TCC_Free;
+  }
+
+  // Find the cost of the histogram operation itself.
+  Type *PtrTy = VectorType::get(AddressTy, VF);
+  Type *MaskTy = VectorType::get(Type::getInt1Ty(Ctx.LLVMCtx), VF);
+  IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
+                              Type::getVoidTy(Ctx.LLVMCtx),
+                              {PtrTy, IncTy, MaskTy});
+
+  // Add the costs together with the add/sub operation.
+  return Ctx.TTI.getIntrinsicInstrCost(
+             ICA, TargetTransformInfo::TCK_RecipThroughput) +
+         MulCost + Ctx.TTI.getArithmeticInstrCost(Opcode, VTy);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPHistogramRecipe::print(raw_ostream &O, const Twine &Indent,
+                              VPSlotTracker &SlotTracker) const {
+  O << Indent << "WIDEN-HISTOGRAM buckets: ";
+  getOperand(0)->printAsOperand(O, SlotTracker);
+
+  if (Opcode == Instruction::Sub)
+    O << ", dec: ";
+  else {
+    assert(Opcode == Instruction::Add);
+    O << ", inc: ";
+  }
+  getOperand(1)->printAsOperand(O, SlotTracker);
+
+  if (VPValue *Mask = getMask()) {
+    O << ", mask: ";
+    Mask->printAsOperand(O, SlotTracker);
+  }
+}
 
 void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
                                 VPSlotTracker &SlotTracker) const {
@@ -1739,7 +1832,7 @@ void VPVectorPointerRecipe ::execute(VPTransformState &State) {
   // or query DataLayout for a more suitable index type otherwise.
   const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();
   Type *IndexTy = State.VF.isScalable() && (IsReverse || CurrentPart > 0)
-                      ? DL.getIndexType(IndexedTy->getPointerTo())
+                      ? DL.getIndexType(Builder.getPtrTy(0))
                       : Builder.getInt32Ty();
   Value *Ptr = State.get(getOperand(0), VPLane(0));
   bool InBounds = isInBounds();
@@ -2267,6 +2360,31 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
   State.set(this, Res);
 }
 
+InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF,
+                                                  VPCostContext &Ctx) const {
+  if (!Consecutive || IsMasked)
+    return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+
+  // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+  // here because the EVL recipes using EVL to replace the tail mask. But in the
+  // legacy model, it will always calculate the cost of mask.
+  // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+  // don't need to compare to the legacy cost model.
+  Type *Ty = ToVectorTy(getLoadStoreType(&Ingredient), VF);
+  const Align Alignment =
+      getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
+  unsigned AS =
+      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
+      Ingredient.getOpcode(), Ty, Alignment, AS, CostKind);
+  if (!Reverse)
+    return Cost;
+
+  return Cost + Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                       cast<VectorType>(Ty), {}, CostKind, 0);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenLoadEVLRecipe::print(raw_ostream &O, const Twine &Indent,
                                  VPSlotTracker &SlotTracker) const {
@@ -2363,6 +2481,31 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
   State.addMetadata(NewSI, SI);
 }
 
+InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF,
+                                                   VPCostContext &Ctx) const {
+  if (!Consecutive || IsMasked)
+    return VPWidenMemoryRecipe::computeCost(VF, Ctx);
+
+  // We need to use the getMaskedMemoryOpCost() instead of getMemoryOpCost()
+  // here because the EVL recipes using EVL to replace the tail mask. But in the
+  // legacy model, it will always calculate the cost of mask.
+  // TODO: Using getMemoryOpCost() instead of getMaskedMemoryOpCost when we
+  // don't need to compare to the legacy cost model.
+  Type *Ty = ToVectorTy(getLoadStoreType(&Ingredient), VF);
+  const Align Alignment =
+      getLoadStoreAlignment(const_cast<Instruction *>(&Ingredient));
+  unsigned AS =
+      getLoadStoreAddressSpace(const_cast<Instruction *>(&Ingredient));
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost(
+      Ingredient.getOpcode(), Ty, Alignment, AS, CostKind);
+  if (!Reverse)
+    return Cost;
+
+  return Cost + Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
+                                       cast<VectorType>(Ty), {}, CostKind, 0);
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenStoreEVLRecipe::print(raw_ostream &O, const Twine &Indent,
                                   VPSlotTracker &SlotTracker) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 4907d3f039727..ca78f32506ef7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -264,24 +264,6 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
     return;
 
   if (auto *VPI = dyn_cast<VPInstruction>(&R)) {
-    VPValue *Op0, *Op1;
-    if (match(VPI, m_VPInstruction<VPInstruction::ExtractFromEnd>(
-                       m_VPValue(Op0), m_VPValue(Op1)))) {
-      VPI->setOperand(1, getValueForPart(Op1, UF - 1));
-      addUniformForAllParts(VPI);
-      if (Plan.hasScalarVFOnly()) {
-        // Extracting from end with VF = 1 implies retrieving the scalar part UF
-        // - Op1.
-        unsigned Offset =
-            cast<ConstantInt>(Op1->getLiveInIRValue())->getZExtValue();
-        VPI->replaceAllUsesWith(getValueForPart(Op0, UF - Offset));
-      } else {
-        // Otherwise we extract from the last part.
-        remapOperands(VPI, UF - 1);
-      }
-      return;
-    }
-
     if (vputils::onlyFirstPartUsed(VPI)) {
       addUniformForAllParts(VPI);
       return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a23a59aa2f11c..4c383244f96f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -358,6 +358,7 @@ class VPDef {
     VPWidenEVLSC,
     VPWidenSelectSC,
     VPBlendSC,
+    VPHistogramSC,
     // START: Phi-like recipes. Need to be kept together.
     VPWidenPHISC,
     VPPredInstPHISC,
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 187c44fb9d04d..d948b7eb39b39 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -559,6 +559,9 @@ if(build_runtimes)
     if(TARGET clang-offload-packager)
       list(APPEND extra_deps clang-offload-packager)
     endif()
+    if(TARGET clang-nvlink-wrapper)
+      list(APPEND extra_deps clang-nvlink-wrapper)
+    endif()
   endif()
   if(LLVM_LIBC_FULL_BUILD)
     list(APPEND extra_cmake_args "-DLLVM_LIBC_FULL_BUILD=ON")
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index 5650d2cf90eac..853eef6bcb2e8 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -93,7 +93,7 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i16 = call <32 x i16> @llvm.vp.sext.v32i16.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i8_v32i32 = call <32 x i32> @llvm.vp.sext.v32i32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32i8_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
@@ -103,7 +103,7 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i1_v32i8 = call <32 x i8> @llvm.vp.sext.v32i8.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i1_v32i16 = call <32 x i16> @llvm.vp.sext.v32i16.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i1_v32i32 = call <32 x i32> @llvm.vp.sext.v32i32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
@@ -112,8 +112,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i16 = call <64 x i16> @llvm.vp.sext.v64i16.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64i8_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v64i8_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
@@ -122,8 +122,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64i32_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i1_v64i8 = call <64 x i8> @llvm.vp.sext.v64i8.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i1_v64i16 = call <64 x i16> @llvm.vp.sext.v64i16.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
@@ -131,9 +131,9 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v128i8_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v128i8_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_v128i8_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
@@ -141,29 +141,29 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v128i16_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128i32_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i1_v128i8 = call <128 x i8> @llvm.vp.sext.v128i8.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v256i8_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v256i8_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %vp_v256i8_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v256i16_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %vp_v256i16_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %vp_v256i32_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.sext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.sext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -253,7 +253,7 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i8_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.sext.nxv16i16.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i8_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.sext.nxv16i32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16i8_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
@@ -263,7 +263,7 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i1_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.sext.nxv16i8.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i1_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.sext.nxv16i16.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16i1_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.sext.nxv16i32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
@@ -272,8 +272,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i8_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.sext.nxv32i16.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32i8_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv32i8_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
@@ -282,8 +282,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32i32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i1_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.sext.nxv32i8.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv32i1_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.sext.nxv32i16.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
@@ -291,8 +291,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv64i8_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv64i8_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
@@ -301,8 +301,8 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_nxv64i16_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %vp_nxv64i32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv64i1_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.sext.nxv64i8.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
@@ -310,9 +310,9 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv128i8_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv128i8_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
@@ -320,9 +320,9 @@ define void @sext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv128i16_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i16_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i32_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i32(<vscale x 128 x i32> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.sext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.sext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i1_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -416,7 +416,7 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = sext <32 x i1> undef to <32 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = sext <32 x i1> undef to <32 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = sext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i1_v32i64 = sext <32 x i1> undef to <32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i16 = call <32 x i16> @llvm.vp.sext.v32i16.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i8_v32i32 = call <32 x i32> @llvm.vp.sext.v32i32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32i8_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
@@ -426,7 +426,7 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i1_v32i8 = call <32 x i8> @llvm.vp.sext.v32i8.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i1_v32i16 = call <32 x i16> @llvm.vp.sext.v32i16.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i1_v32i32 = call <32 x i32> @llvm.vp.sext.v32i32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.sext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = sext <64 x i8> undef to <64 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = sext <64 x i8> undef to <64 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = sext <64 x i8> undef to <64 x i64>
@@ -435,8 +435,8 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = sext <64 x i32> undef to <64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = sext <64 x i1> undef to <64 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = sext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i1_v64i32 = sext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v64i1_v64i64 = sext <64 x i1> undef to <64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i16 = call <64 x i16> @llvm.vp.sext.v64i16.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64i8_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v64i8_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
@@ -445,8 +445,8 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64i32_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i1_v64i8 = call <64 x i8> @llvm.vp.sext.v64i8.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i1_v64i16 = call <64 x i16> @llvm.vp.sext.v64i16.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.sext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.sext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = sext <128 x i8> undef to <128 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = sext <128 x i8> undef to <128 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = sext <128 x i8> undef to <128 x i64>
@@ -454,9 +454,9 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = sext <128 x i16> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = sext <128 x i32> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = sext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v128i1_v128i16 = sext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v128i1_v128i32 = sext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v128i1_v128i64 = sext <128 x i1> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v128i8_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v128i8_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_v128i8_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
@@ -464,29 +464,29 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v128i16_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128i32_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i1_v128i8 = call <128 x i8> @llvm.vp.sext.v128i8.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.sext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.sext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.sext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = sext <256 x i8> undef to <256 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = sext <256 x i8> undef to <256 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = sext <256 x i8> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = sext <256 x i16> undef to <256 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = sext <256 x i16> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = sext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v256i1_v256i8 = sext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v256i1_v256i16 = sext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v256i1_v256i32 = sext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v256i1_v256i64 = sext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v256i8_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v256i8_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %vp_v256i8_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v256i16_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %vp_v256i16_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %vp_v256i32_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.sext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.sext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.sext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.sext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.sext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = sext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = sext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = sext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -576,7 +576,7 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = sext <vscale x 16 x i1> undef to <vscale x 16 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = sext <vscale x 16 x i1> undef to <vscale x 16 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = sext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i1_nxv16i64 = sext <vscale x 16 x i1> undef to <vscale x 16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i8_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.sext.nxv16i16.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i8_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.sext.nxv16i32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16i8_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
@@ -586,7 +586,7 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i1_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.sext.nxv16i8.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i1_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.sext.nxv16i16.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16i1_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.sext.nxv16i32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.sext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = sext <vscale x 32 x i8> undef to <vscale x 32 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = sext <vscale x 32 x i8> undef to <vscale x 32 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = sext <vscale x 32 x i8> undef to <vscale x 32 x i64>
@@ -595,8 +595,8 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = sext <vscale x 32 x i32> undef to <vscale x 32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = sext <vscale x 32 x i1> undef to <vscale x 32 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = sext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i1_nxv32i32 = sext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv32i1_nxv32i64 = sext <vscale x 32 x i1> undef to <vscale x 32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i8_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.sext.nxv32i16.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32i8_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv32i8_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
@@ -605,8 +605,8 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32i32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i1_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.sext.nxv32i8.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv32i1_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.sext.nxv32i16.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.sext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.sext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = sext <vscale x 64 x i8> undef to <vscale x 64 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = sext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = sext <vscale x 64 x i8> undef to <vscale x 64 x i64>
@@ -614,9 +614,9 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = sext <vscale x 64 x i16> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = sext <vscale x 64 x i32> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = sext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv64i1_nxv64i16 = sext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv64i1_nxv64i32 = sext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv64i1_nxv64i64 = sext <vscale x 64 x i1> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv64i8_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv64i8_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_nxv64i8_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
@@ -624,18 +624,18 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv64i16_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64i32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv64i1_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.sext.nxv64i8.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.sext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.sext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.sext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = sext <vscale x 128 x i8> undef to <vscale x 128 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = sext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = sext <vscale x 128 x i8> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = sext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = sext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = sext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv128i1_nxv128i8 = sext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv128i1_nxv128i16 = sext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv128i1_nxv128i32 = sext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = sext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv128i8_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv128i8_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
@@ -643,9 +643,9 @@ define void @sext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv128i16_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i16_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i32_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i32(<vscale x 128 x i32> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.sext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.sext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.sext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.sext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i1_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.sext.nxv128i128.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -1095,7 +1095,7 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i16 = call <32 x i16> @llvm.vp.zext.v32i16.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i8_v32i32 = call <32 x i32> @llvm.vp.zext.v32i32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32i8_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
@@ -1105,7 +1105,7 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i1_v32i8 = call <32 x i8> @llvm.vp.zext.v32i8.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i1_v32i16 = call <32 x i16> @llvm.vp.zext.v32i16.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i1_v32i32 = call <32 x i32> @llvm.vp.zext.v32i32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
@@ -1114,8 +1114,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i16 = call <64 x i16> @llvm.vp.zext.v64i16.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64i8_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v64i8_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
@@ -1124,8 +1124,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64i32_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i1_v64i8 = call <64 x i8> @llvm.vp.zext.v64i8.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i1_v64i16 = call <64 x i16> @llvm.vp.zext.v64i16.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
@@ -1133,9 +1133,9 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v128i8_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v128i8_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_v128i8_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
@@ -1143,29 +1143,29 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v128i16_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128i32_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i1_v128i8 = call <128 x i8> @llvm.vp.zext.v128i8.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v256i8_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v256i8_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %vp_v256i8_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v256i16_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %vp_v256i16_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %vp_v256i32_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.zext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.zext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -1255,7 +1255,7 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i8_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.zext.nxv16i16.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i8_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.zext.nxv16i32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16i8_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
@@ -1265,7 +1265,7 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i1_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.zext.nxv16i8.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i1_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.zext.nxv16i16.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16i1_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.zext.nxv16i32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
@@ -1274,8 +1274,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i8_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.zext.nxv32i16.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32i8_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv32i8_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
@@ -1284,8 +1284,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32i32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i1_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.zext.nxv32i8.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv32i1_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.zext.nxv32i16.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
@@ -1293,8 +1293,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv64i8_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv64i8_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
@@ -1303,8 +1303,8 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_nxv64i16_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %vp_nxv64i32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv64i1_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.zext.nxv64i8.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
@@ -1312,9 +1312,9 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv128i8_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv128i8_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
@@ -1322,9 +1322,9 @@ define void @zext() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv128i16_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i16_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i32_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i32(<vscale x 128 x i32> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.zext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.zext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i1_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -1418,7 +1418,7 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i1_v32i8 = zext <32 x i1> undef to <32 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i1_v32i16 = zext <32 x i1> undef to <32 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32i32 = zext <32 x i1> undef to <32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i1_v32i64 = zext <32 x i1> undef to <32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i16 = call <32 x i16> @llvm.vp.zext.v32i16.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i8_v32i32 = call <32 x i32> @llvm.vp.zext.v32i32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32i8_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
@@ -1428,7 +1428,7 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i1_v32i8 = call <32 x i8> @llvm.vp.zext.v32i8.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i1_v32i16 = call <32 x i16> @llvm.vp.zext.v32i16.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i1_v32i32 = call <32 x i32> @llvm.vp.zext.v32i32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32i1_v32i64 = call <32 x i64> @llvm.vp.zext.v32i64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i16 = zext <64 x i8> undef to <64 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i8_v64i32 = zext <64 x i8> undef to <64 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v64i8_v64i64 = zext <64 x i8> undef to <64 x i64>
@@ -1437,8 +1437,8 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64i64 = zext <64 x i32> undef to <64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i1_v64i8 = zext <64 x i1> undef to <64 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i1_v64i16 = zext <64 x i1> undef to <64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i1_v64i32 = zext <64 x i1> undef to <64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v64i1_v64i64 = zext <64 x i1> undef to <64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i16 = call <64 x i16> @llvm.vp.zext.v64i16.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64i8_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v64i8_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
@@ -1447,8 +1447,8 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64i32_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i1_v64i8 = call <64 x i8> @llvm.vp.zext.v64i8.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i1_v64i16 = call <64 x i16> @llvm.vp.zext.v64i16.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64i1_v64i32 = call <64 x i32> @llvm.vp.zext.v64i32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v64i1_v64i64 = call <64 x i64> @llvm.vp.zext.v64i64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v128i8_v128i16 = zext <128 x i8> undef to <128 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %v128i8_v128i32 = zext <128 x i8> undef to <128 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %v128i8_v128i64 = zext <128 x i8> undef to <128 x i64>
@@ -1456,9 +1456,9 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v128i16_v128i64 = zext <128 x i16> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128i64 = zext <128 x i32> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i1_v128i8 = zext <128 x i1> undef to <128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v128i1_v128i16 = zext <128 x i1> undef to <128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v128i1_v128i32 = zext <128 x i1> undef to <128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v128i1_v128i64 = zext <128 x i1> undef to <128 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v128i8_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_v128i8_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_v128i8_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
@@ -1466,29 +1466,29 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v128i16_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128i32_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i1_v128i8 = call <128 x i8> @llvm.vp.zext.v128i8.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v128i1_v128i16 = call <128 x i16> @llvm.vp.zext.v128i16.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v128i1_v128i32 = call <128 x i32> @llvm.vp.zext.v128i32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v128i1_v128i64 = call <128 x i64> @llvm.vp.zext.v128i64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v256i8_v256i16 = zext <256 x i8> undef to <256 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %v256i8_v256i32 = zext <256 x i8> undef to <256 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %v256i8_v256i64 = zext <256 x i8> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v256i16_v256i32 = zext <256 x i16> undef to <256 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %v256i16_v256i64 = zext <256 x i16> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %v256i32_v256i64 = zext <256 x i32> undef to <256 x i64>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v256i1_v256i8 = zext <256 x i1> undef to <256 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %v256i1_v256i16 = zext <256 x i1> undef to <256 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v256i1_v256i32 = zext <256 x i1> undef to <256 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %v256i1_v256i64 = zext <256 x i1> undef to <256 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v256i8_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_v256i8_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %vp_v256i8_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v256i16_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 140 for instruction: %vp_v256i16_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %vp_v256i32_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.zext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 270 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v256i1_v256i8 = call <256 x i8> @llvm.vp.zext.v256i8.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_v256i1_v256i16 = call <256 x i16> @llvm.vp.zext.v256i16.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v256i1_v256i32 = call <256 x i32> @llvm.vp.zext.v256i32.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 151 for instruction: %vp_v256i1_v256i64 = call <256 x i64> @llvm.vp.zext.v256i64.v256i1(<256 x i1> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i16 = zext <vscale x 1 x i8> undef to <vscale x 1 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i32 = zext <vscale x 1 x i8> undef to <vscale x 1 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8_nxv1i64 = zext <vscale x 1 x i8> undef to <vscale x 1 x i64>
@@ -1578,7 +1578,7 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i1_nxv16i8 = zext <vscale x 16 x i1> undef to <vscale x 16 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i1_nxv16i16 = zext <vscale x 16 x i1> undef to <vscale x 16 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16i32 = zext <vscale x 16 x i1> undef to <vscale x 16 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i1_nxv16i64 = zext <vscale x 16 x i1> undef to <vscale x 16 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i8_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.zext.nxv16i16.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i8_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.zext.nxv16i32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16i8_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
@@ -1588,7 +1588,7 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16i1_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.zext.nxv16i8.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16i1_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.zext.nxv16i16.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16i1_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.zext.nxv16i32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16i1_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.zext.nxv16i64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i8_nxv32i16 = zext <vscale x 32 x i8> undef to <vscale x 32 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i8_nxv32i32 = zext <vscale x 32 x i8> undef to <vscale x 32 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv32i8_nxv32i64 = zext <vscale x 32 x i8> undef to <vscale x 32 x i64>
@@ -1597,8 +1597,8 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32i64 = zext <vscale x 32 x i32> undef to <vscale x 32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv32i1_nxv32i8 = zext <vscale x 32 x i1> undef to <vscale x 32 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv32i1_nxv32i16 = zext <vscale x 32 x i1> undef to <vscale x 32 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i1_nxv32i32 = zext <vscale x 32 x i1> undef to <vscale x 32 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv32i1_nxv32i64 = zext <vscale x 32 x i1> undef to <vscale x 32 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i8_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.zext.nxv32i16.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32i8_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv32i8_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
@@ -1607,8 +1607,8 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32i32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv32i1_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.zext.nxv32i8.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv32i1_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.zext.nxv32i16.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32i1_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.zext.nxv32i32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv32i1_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.zext.nxv32i64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv64i8_nxv64i16 = zext <vscale x 64 x i8> undef to <vscale x 64 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %nxv64i8_nxv64i32 = zext <vscale x 64 x i8> undef to <vscale x 64 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %nxv64i8_nxv64i64 = zext <vscale x 64 x i8> undef to <vscale x 64 x i64>
@@ -1616,9 +1616,9 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv64i16_nxv64i64 = zext <vscale x 64 x i16> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64i64 = zext <vscale x 64 x i32> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv64i1_nxv64i8 = zext <vscale x 64 x i1> undef to <vscale x 64 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv64i1_nxv64i16 = zext <vscale x 64 x i1> undef to <vscale x 64 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv64i1_nxv64i32 = zext <vscale x 64 x i1> undef to <vscale x 64 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv64i1_nxv64i64 = zext <vscale x 64 x i1> undef to <vscale x 64 x i64>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv64i8_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %vp_nxv64i8_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %vp_nxv64i8_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
@@ -1626,18 +1626,18 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv64i16_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64i32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv64i1_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.zext.nxv64i8.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv64i1_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.zext.nxv64i16.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv64i1_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.zext.nxv64i32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv64i1_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.zext.nxv64i64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv128i8_nxv128i16 = zext <vscale x 128 x i8> undef to <vscale x 128 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %nxv128i8_nxv128i32 = zext <vscale x 128 x i8> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i8_nxv128i128 = zext <vscale x 128 x i8> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv128i16_nxv128i32 = zext <vscale x 128 x i16> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i16_nxv128i128 = zext <vscale x 128 x i16> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i32_nxv128i128 = zext <vscale x 128 x i32> undef to <vscale x 128 x i128>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv128i1_nxv128i8 = zext <vscale x 128 x i1> undef to <vscale x 128 x i8>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %nxv128i1_nxv128i16 = zext <vscale x 128 x i1> undef to <vscale x 128 x i16>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %nxv128i1_nxv128i32 = zext <vscale x 128 x i1> undef to <vscale x 128 x i32>
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %nxv128i1_nxv128i128 = zext <vscale x 128 x i1> undef to <vscale x 128 x i128>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv128i8_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %vp_nxv128i8_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i8(<vscale x 128 x i8> undef, <vscale x 128 x i1> undef, i32 undef)
@@ -1645,9 +1645,9 @@ define void @zext() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv128i16_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i16_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i16(<vscale x 128 x i16> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i32_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i32(<vscale x 128 x i32> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.zext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv128i1_nxv128i8 = call <vscale x 128 x i8> @llvm.vp.zext.nxv128i8.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %vp_nxv128i1_nxv128i16 = call <vscale x 128 x i16> @llvm.vp.zext.nxv128i16.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_nxv128i1_nxv128i32 = call <vscale x 128 x i32> @llvm.vp.zext.nxv128i32.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv128i1_nxv128i128 = call <vscale x 128 x i128> @llvm.vp.zext.nxv128i128.nxv128i1(<vscale x 128 x i1> undef, <vscale x 128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
@@ -2117,7 +2117,7 @@ define void @trunc() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i16_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i32_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %vp_v32i64_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i64_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8>
@@ -2127,7 +2127,7 @@ define void @trunc() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64i16_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64i32_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64i64_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
@@ -2137,7 +2137,7 @@ define void @trunc() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i16_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i32_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v64i64_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i64_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8>
@@ -2147,7 +2147,7 @@ define void @trunc() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v128i16_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128i32_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128i64_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
@@ -2157,27 +2157,27 @@ define void @trunc() {
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i8_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i16_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i32_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v128i64_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i64_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1>
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1>
+; RV32-NEXT:  Cost Model: Found an estimated cost of 271 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v256i16_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v256i32_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v256i64_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v256i32_v256i16 = call <256 x i16> @llvm.vp.trunc.v256i16.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %vp_v256i64_v256i16 = call <256 x i16> @llvm.vp.trunc.v256i16.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %vp_v256i64_v256i32 = call <256 x i32> @llvm.vp.trunc.v256i32.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i8_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i16_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 134 for instruction: %vp_v256i32_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV32-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v256i64_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v256i8_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v256i16_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v256i32_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
+; RV32-NEXT:  Cost Model: Found an estimated cost of 271 for instruction: %vp_v256i64_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1i8 = trunc <vscale x 1 x i16> undef to <vscale x 1 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i32_nxv1i8 = trunc <vscale x 1 x i32> undef to <vscale x 1 x i8>
 ; RV32-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i64_nxv1i8 = trunc <vscale x 1 x i64> undef to <vscale x 1 x i8>
@@ -2430,7 +2430,7 @@ define void @trunc() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32i8_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32i16_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32i32_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %vp_v32i64_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32i64_v32i1 = call <32 x i1> @llvm.vp.trunc.v32i1.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64i16_v64i8 = trunc <64 x i16> undef to <64 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64i32_v64i8 = trunc <64 x i32> undef to <64 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64i64_v64i8 = trunc <64 x i64> undef to <64 x i8>
@@ -2439,8 +2439,8 @@ define void @trunc() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64i32 = trunc <64 x i64> undef to <64 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8_v64i1 = trunc <64 x i8> undef to <64 x i1>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i16_v64i1 = trunc <64 x i16> undef to <64 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i32_v64i1 = trunc <64 x i32> undef to <64 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i64_v64i1 = trunc <64 x i64> undef to <64 x i1>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64i16_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64i32_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64i64_v64i8 = call <64 x i8> @llvm.vp.trunc.v64i8.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
@@ -2449,8 +2449,8 @@ define void @trunc() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64i64_v64i32 = call <64 x i32> @llvm.vp.trunc.v64i32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v64i8_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64i16_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %vp_v64i32_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %vp_v64i64_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64i32_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64i64_v64i1 = call <64 x i1> @llvm.vp.trunc.v64i1.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v128i16_v128i8 = trunc <128 x i16> undef to <128 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128i32_v128i8 = trunc <128 x i32> undef to <128 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128i64_v128i8 = trunc <128 x i64> undef to <128 x i8>
@@ -2459,8 +2459,8 @@ define void @trunc() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128i32 = trunc <128 x i64> undef to <128 x i32>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v128i8_v128i1 = trunc <128 x i8> undef to <128 x i1>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v128i16_v128i1 = trunc <128 x i16> undef to <128 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i32_v128i1 = trunc <128 x i32> undef to <128 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i64_v128i1 = trunc <128 x i64> undef to <128 x i1>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v128i16_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128i32_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128i64_v128i8 = call <128 x i8> @llvm.vp.trunc.v128i8.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
@@ -2469,28 +2469,28 @@ define void @trunc() {
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128i64_v128i32 = call <128 x i32> @llvm.vp.trunc.v128i32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v128i8_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v128i16_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v128i32_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v128i64_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128i32_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128i64_v128i1 = call <128 x i1> @llvm.vp.trunc.v128i1.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v256i16_v256i8 = trunc <256 x i16> undef to <256 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v256i32_v256i8 = trunc <256 x i32> undef to <256 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v256i64_v256i8 = trunc <256 x i64> undef to <256 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v256i32_v256i16 = trunc <256 x i32> undef to <256 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %v256i64_v256i16 = trunc <256 x i64> undef to <256 x i16>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %v256i64_v256i32 = trunc <256 x i64> undef to <256 x i32>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1>
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v256i8_v256i1 = trunc <256 x i8> undef to <256 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v256i16_v256i1 = trunc <256 x i16> undef to <256 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v256i32_v256i1 = trunc <256 x i32> undef to <256 x i1>
+; RV64-NEXT:  Cost Model: Found an estimated cost of 271 for instruction: %v256i64_v256i1 = trunc <256 x i64> undef to <256 x i1>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v256i16_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v256i32_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v256i64_v256i8 = call <256 x i8> @llvm.vp.trunc.v256i8.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v256i32_v256i16 = call <256 x i16> @llvm.vp.trunc.v256i16.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %vp_v256i64_v256i16 = call <256 x i16> @llvm.vp.trunc.v256i16.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %vp_v256i64_v256i32 = call <256 x i32> @llvm.vp.trunc.v256i32.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v256i8_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %vp_v256i16_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v256i32_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
-; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v256i64_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v256i8_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i8(<256 x i8> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v256i16_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i16(<256 x i16> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v256i32_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i32(<256 x i32> undef, <256 x i1> undef, i32 undef)
+; RV64-NEXT:  Cost Model: Found an estimated cost of 271 for instruction: %vp_v256i64_v256i1 = call <256 x i1> @llvm.vp.trunc.v256i1.v256i64(<256 x i64> undef, <256 x i1> undef, i32 undef)
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1i8 = trunc <vscale x 1 x i16> undef to <vscale x 1 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i32_nxv1i8 = trunc <vscale x 1 x i32> undef to <vscale x 1 x i8>
 ; RV64-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i64_nxv1i8 = trunc <vscale x 1 x i64> undef to <vscale x 1 x i8>
@@ -6959,3 +6959,29 @@ define void @oddvec_sizes() {
 
   ret void
 }
+
+; This test exercises cases which involve both vector widening and
+; vector splitting.  We previously crashed on this case due to an
+; infinite recursion between cast costing and scalarization costing.
+define void @legalization_crash() {
+; CHECK-LABEL: 'legalization_crash'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16700 for instruction: %1 = bitcast <24 x i8> undef to <192 x i1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %2 = trunc <192 x i8> undef to <192 x i1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %3 = zext <192 x i1> undef to <192 x i8>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %4 = sext <192 x i1> undef to <192 x i8>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %5 = sitofp <192 x i1> undef to <192 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %6 = uitofp <192 x i1> undef to <192 x float>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %7 = fptosi <192 x float> undef to <192 x i1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %8 = fptoui <192 x float> undef to <192 x i1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  bitcast <24 x i8> undef to <192 x i1>
+  trunc <192 x i8> undef to <192 x i1>
+  zext <192 x i1> undef to <192 x i8>
+  sext <192 x i1> undef to <192 x i8>
+  sitofp <192 x i1> undef to <192 x float>
+  uitofp <192 x i1> undef to <192 x float>
+  fptosi <192 x float> undef to <192 x i1>
+  fptoui <192 x float> undef to <192 x i1>
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
index 118d92b80d254..fce9f71d37634 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cttz_elts.ll
@@ -9,28 +9,28 @@ define void @foo_no_vscale_range() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 781 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 662 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 335 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 781 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 662 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 390 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 335 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
@@ -75,28 +75,28 @@ define void @foo_vscale_range_2_16() vscale_range(2,16) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv16i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv32i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i64.nxv64i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i64.nxv128i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv2i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv4i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv8i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv16i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv32i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res.i32.nxv64i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 true)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i32.nxv128i1.zip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 true)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv2i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv4i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv8i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv16i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv32i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i64.nxv64i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i64.nxv128i1.nzip = call i64 @llvm.experimental.cttz.elts.i64.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv2i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv2i1(<vscale x 2 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv4i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv4i1(<vscale x 4 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv8i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv8i1(<vscale x 8 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv16i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv32i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv32i1(<vscale x 32 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res.i32.nxv64i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv64i1(<vscale x 64 x i1> undef, i1 false)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 195 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %res.i32.nxv128i1.nzip = call i32 @llvm.experimental.cttz.elts.i32.nxv128i1(<vscale x 128 x i1> undef, i1 false)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %res.i64.nxv2i1.zip = call i64 @llvm.experimental.cttz.elts.i64.nxv2i1(<vscale x 2 x i1> undef, i1 true)
diff --git a/llvm/test/Analysis/CtxProfAnalysis/flatten-icp.ll b/llvm/test/Analysis/CtxProfAnalysis/flatten-icp.ll
new file mode 100644
index 0000000000000..fbffe780f0afa
--- /dev/null
+++ b/llvm/test/Analysis/CtxProfAnalysis/flatten-icp.ll
@@ -0,0 +1,55 @@
+; RUN: split-file %s %t
+; RUN: llvm-ctxprof-util fromJSON --input %t/profile.json --output %t/profile.ctxprofdata
+;
+; In the given profile, in one of the contexts the indirect call is taken, the
+; target we're trying to ICP - GUID:2000 - doesn't appear at all. That should
+; contribute to the count of the "indirect call BB".
+; RUN: opt %t/test.ll -S -passes='require<ctx-prof-analysis>,module-inline,ctx-prof-flatten' -use-ctx-profile=%t/profile.ctxprofdata -ctx-prof-promote-alwaysinline 
+
+; CHECK-LABEL: define i32 @caller(ptr %c)
+; CHECK-NEXT:     [[CND:[0-9]+]] = icmp eq ptr %c, @one
+; CHECK-NEXT:     br i1 [[CND]], label %{{.*}}, label %{{.*}}, !prof ![[BW:[0-9]+]]
+
+; CHECK: ![[BW]] = !{!"branch_weights", i32 10, i32 10}
+
+;--- test.ll
+declare i32 @external(i32 %x)
+define i32 @one() #0 !guid !0 {
+  call void @llvm.instrprof.increment(ptr @one, i64 123, i32 1, i32 0)
+  call void @llvm.instrprof.callsite(ptr @one, i64 123, i32 1, i32 0, ptr @external)
+  %ret = call i32 @external(i32 1)
+  ret i32 %ret
+}
+
+define i32 @caller(ptr %c) #1 !guid !1 {
+  call void @llvm.instrprof.increment(ptr @caller, i64 567, i32 1, i32 0)
+  call void @llvm.instrprof.callsite(ptr @caller, i64 567, i32 1, i32 0, ptr %c)
+  %ret = call i32 %c()
+  ret i32 %ret
+}
+
+define i32 @root(ptr %c) !guid !2 {
+  call void @llvm.instrprof.increment(ptr @root, i64 432, i32 1, i32 0)
+  call void @llvm.instrprof.callsite(ptr @root, i64 432, i32 2, i32 0, ptr @caller)
+  %a = call i32 @caller(ptr %c)
+  call void @llvm.instrprof.callsite(ptr @root, i64 432, i32 2, i32 1, ptr @caller)
+  %b = call i32 @caller(ptr %c)
+  %ret = add i32 %a, %b
+  ret i32 %ret
+
+}
+
+attributes #0 = { alwaysinline }
+attributes #1 = { noinline }
+!0 = !{i64 1000}
+!1 = !{i64 3000}
+!2 = !{i64 4000}
+
+;--- profile.json
+[ {
+  "Guid": 4000, "Counters":[10], "Callsites": [
+    [{"Guid":3000, "Counters":[10], "Callsites":[[{"Guid":1000, "Counters":[10]}]]}],
+    [{"Guid":3000, "Counters":[10], "Callsites":[[{"Guid":9000, "Counters":[10]}]]}]
+  ]
+}
+]
diff --git a/llvm/test/Analysis/Lint/const-store.ll b/llvm/test/Analysis/Lint/const-store.ll
new file mode 100644
index 0000000000000..030a0be3aecc2
--- /dev/null
+++ b/llvm/test/Analysis/Lint/const-store.ll
@@ -0,0 +1,49 @@
+; RUN: not opt --mtriple=amdgcn --passes=lint --lint-abort-on-error %s -disable-output 2>&1 | FileCheck %s
+; RUN: opt --mtriple=amdgcn --mcpu=gfx1030 --passes=lint %s -disable-output 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK0
+; RUN: opt --mtriple=x86_64 --passes=lint --lint-abort-on-error %s -disable-output 2>&1 | FileCheck %s --allow-empty --check-prefix=NOERR
+; NOERR: {{^$}}
+
+define amdgpu_kernel void @store_const(ptr addrspace(4) %out, i32 %a, i32 %b) {
+; CHECK: Undefined behavior: Write to memory in const addrspace
+; CHECK-NEXT: store i32 %r, ptr addrspace(4) %out
+  %r = add i32 %a, %b
+  store i32 %r, ptr addrspace(4) %out
+  ret void
+}
+
+declare void @llvm.memset.p4.i64(ptr addrspace(4) noalias nocapture writeonly, i8, i64, i1)
+define amdgpu_kernel void @memset_const(ptr addrspace(4) %dst) {
+; CHECK0: Undefined behavior: Write to memory in const addrspace
+; CHECK0-NEXT: call void @llvm.memset.p4.i64(ptr addrspace(4) %dst, i8 0, i64 256, i1 false)
+  call void @llvm.memset.p4.i64(ptr addrspace(4) %dst, i8 0, i64 256, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p6.p0.i32(ptr addrspace(6) noalias nocapture writeonly, ptr noalias nocapture readonly, i32, i1)
+define amdgpu_kernel void @memcpy_to_const(ptr addrspace(6) %dst, ptr %src) {
+; CHECK0: Undefined behavior: Write to memory in const addrspace
+; CHECK0-NEXT: call void @llvm.memcpy.p6.p0.i32(ptr addrspace(6) %dst, ptr %src, i32 256, i1 false)
+  call void @llvm.memcpy.p6.p0.i32(ptr addrspace(6) %dst, ptr %src, i32 256, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @cmpxchg_to_const(ptr addrspace(4) %dst, i32 %src) {
+; CHECK0: Undefined behavior: Write to memory in const addrspace
+; CHECK0-NEXT: %void = cmpxchg ptr addrspace(4) %dst, i32 0, i32 %src seq_cst monotonic
+  %void = cmpxchg ptr addrspace(4) %dst, i32 0, i32 %src seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @atomicrmw_to_const(ptr addrspace(4) %dst, i32 %src) {
+; CHECK0: Undefined behavior: Write to memory in const addrspace
+; CHECK0-NEXT: %void = atomicrmw add ptr addrspace(4) %dst, i32 %src acquire
+  %void = atomicrmw add ptr addrspace(4) %dst, i32 %src acquire
+  ret void
+}
+
+declare void @const_param(ptr addrspace(6))
+define amdgpu_kernel void @call_with_const(ptr addrspace(6) %dst) {
+; CHECK0-NOT: call void @const_param(ptr addrspace(6) %dst)
+  call void @const_param(ptr addrspace(6) %dst)
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
index 37d6584b1e85f..124d895e5aef5 100644
--- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
+++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll
@@ -1568,33 +1568,6 @@ exit:
   ret i32 0
 }
 
-
-define i32 @ptr_induction_ult_2(ptr %a, ptr %b) {
-; CHECK-LABEL: 'ptr_induction_ult_2'
-; CHECK-NEXT:  Classifying expressions for: @ptr_induction_ult_2
-; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ]
-; CHECK-NEXT:    --> {%a,+,4}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
-; CHECK-NEXT:    %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1
-; CHECK-NEXT:    --> {(4 + %a),+,4}<%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
-; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_ult_2
-; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
-; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
-; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
-;
-entry:
-  %cmp.6 = icmp ult ptr %a, %b
-  br i1 %cmp.6, label %loop, label %exit
-
-loop:
-  %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ]
-  %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1
-  %exitcond = icmp eq ptr %ptr.iv, %b
-  br i1 %exitcond, label %exit, label %loop
-
-exit:
-  ret i32 0
-}
-
 define void @ptr_induction_eq_1(ptr %a, ptr %b) {
 ; CHECK-LABEL: 'ptr_induction_eq_1'
 ; CHECK-NEXT:  Classifying expressions for: @ptr_induction_eq_1
@@ -1652,47 +1625,6 @@ exit:
   ret void
 }
 
-; TODO: It feels like we should be able to calculate the symbolic max
-; exit count for the loop.inc block here, in the same way as
-; ptr_induction_eq_1. The problem seems to be in howFarToZero when the
-; ControlsOnlyExit is set to false.
-define void @ptr_induction_early_exit_eq_1(ptr %a, ptr %b, ptr %c) {
-; CHECK-LABEL: 'ptr_induction_early_exit_eq_1'
-; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_1
-; CHECK-NEXT:    %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a, %entry ]
-; CHECK-NEXT:    --> {%a,+,8}<nuw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
-; CHECK-NEXT:    %ld1 = load ptr, ptr %ptr.iv, align 8
-; CHECK-NEXT:    --> %ld1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Variant }
-; CHECK-NEXT:    %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
-; CHECK-NEXT:    --> {(8 + %a),+,8}<nw><%loop> U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %loop: Computable }
-; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1
-; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
-; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
-; CHECK-NEXT:    exit count for loop.inc: ***COULDNOTCOMPUTE***
-; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
-; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
-; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
-; CHECK-NEXT:    symbolic max exit count for loop.inc: ***COULDNOTCOMPUTE***
-;
-entry:
-  %cmp = icmp eq ptr %a, %b
-  br i1 %cmp, label %exit, label %loop
-
-loop:
-  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a, %entry ]
-  %ld1 = load ptr, ptr %ptr.iv, align 8
-  %earlyexitcond = icmp eq ptr %ld1, %c
-  br i1 %earlyexitcond, label %exit, label %loop.inc
-
-loop.inc:
-  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
-  %exitcond = icmp eq ptr %ptr.iv.next, %b
-  br i1 %exitcond, label %exit, label %loop
-
-exit:
-  ret void
-}
-
 define void @ptr_induction_early_exit_eq_2(ptr %a, i64 %n, ptr %c) {
 ; CHECK-LABEL: 'ptr_induction_early_exit_eq_2'
 ; CHECK-NEXT:  Classifying expressions for: @ptr_induction_early_exit_eq_2
@@ -1733,7 +1665,6 @@ exit:
   ret void
 }
 
-
 define void @gep_addrec_nw(ptr %a) {
 ; CHECK-LABEL: 'gep_addrec_nw'
 ; CHECK-NEXT:  Classifying expressions for: @gep_addrec_nw
diff --git a/llvm/test/Analysis/ScalarEvolution/predicated-max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/predicated-max-backedge-taken-count-guard-info.ll
new file mode 100644
index 0000000000000..ba3a5b4cfb3ae
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/predicated-max-backedge-taken-count-guard-info.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<scalar-evolution>' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s
+
+define i32 @ptr_induction_ult(ptr %a, ptr %b) {
+; CHECK-LABEL: 'ptr_induction_ult'
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_ult
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %cmp.6 = icmp ult ptr %a, %b
+  br i1 %cmp.6, label %loop, label %exit
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ]
+  %ptr.iv.next = getelementptr i32, ptr %ptr.iv, i64 1
+  %exitcond = icmp eq ptr %ptr.iv, %b
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret i32 0
+}
+
+define i32 @ptr_induction_ult_3_step_6(ptr %a, ptr %b) {
+; CHECK-LABEL: 'ptr_induction_ult_3_step_6'
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_ult_3_step_6
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %cmp.6 = icmp ult ptr %a, %b
+  br i1 %cmp.6, label %loop, label %exit
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ]
+  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 6
+  %exitcond = icmp eq ptr %ptr.iv, %b
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret i32 0
+}
+
+define i32 @ptr_induction_ult_3_step_7(ptr %a, ptr %b) {
+; CHECK-LABEL: 'ptr_induction_ult_3_step_7'
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_ult_3_step_7
+; CHECK-NEXT:  Loop %loop: backedge-taken count is ((7905747460161236407 * (ptrtoint ptr %b to i64)) + (-7905747460161236407 * (ptrtoint ptr %a to i64)))
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i64 -1
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is ((7905747460161236407 * (ptrtoint ptr %b to i64)) + (-7905747460161236407 * (ptrtoint ptr %a to i64)))
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+entry:
+  %cmp.6 = icmp ult ptr %a, %b
+  br i1 %cmp.6, label %loop, label %exit
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ]
+  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 7
+  %exitcond = icmp eq ptr %ptr.iv, %b
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret i32 0
+}
+
+; %a and %b may not have the same alignment, so the loop may only via the early
+; exit when %ptr.iv > %b. The predicated exit count for the latch can be
+; computed by adding a predicate.
+define void @ptr_induction_early_exit_eq_1(ptr %a, ptr %b, ptr %c) {
+; CHECK-LABEL: 'ptr_induction_early_exit_eq_1'
+; CHECK-NEXT:  Determining loop execution counts for: @ptr_induction_early_exit_eq_1
+; CHECK-NEXT:  Loop %loop: <multiple exits> Unpredictable backedge-taken count.
+; CHECK-NEXT:    exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    exit count for loop.inc: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+; CHECK-NEXT:    symbolic max exit count for loop: ***COULDNOTCOMPUTE***
+; CHECK-NEXT:    symbolic max exit count for loop.inc: ***COULDNOTCOMPUTE***
+;
+entry:
+  %cmp = icmp eq ptr %a, %b
+  br i1 %cmp, label %exit, label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %ptr.iv.next, %loop.inc ], [ %a, %entry ]
+  %ld1 = load ptr, ptr %ptr.iv, align 8
+  %earlyexitcond = icmp eq ptr %ld1, %c
+  br i1 %earlyexitcond, label %exit, label %loop.inc
+
+loop.inc:
+  %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8
+  %exitcond = icmp eq ptr %ptr.iv.next, %b
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-urem.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-urem.ll
new file mode 100644
index 0000000000000..928d9accf35e7
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count-urem.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes='print<scalar-evolution>' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s
+
+declare void @foo()
+
+define void @test_trip_count_expr_contains_urem(i32 %N) {
+; CHECK-LABEL: 'test_trip_count_expr_contains_urem'
+; CHECK-NEXT:  Determining loop execution counts for: @test_trip_count_expr_contains_urem
+; CHECK-NEXT:  Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT:  Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+  %n.rnd.up = add i32 %N, 1
+  %n.mod.vf = urem i32 %n.rnd.up, 16
+  %n.vec = sub i32 %n.rnd.up, %n.mod.vf
+  br label %loop
+
+loop:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %loop ]
+  %index.next = add i32 %index, 16
+  call void @foo()
+  %ec = icmp eq i32 %index, %n.vec
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.image.atomic.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.image.atomic.ll
index f6ac8fbd99c58..e2b743e5f514a 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.image.atomic.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/llvm.amdgcn.image.atomic.ll
@@ -1,131 +1,131 @@
 ; RUN: opt -mtriple amdgcn-mesa-mesa3d -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(
 define float @image_atomic_swap(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32.v8i32(
 define float @image_atomic_add(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32.v8i32(
 define float @image_atomic_sub(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32.v8i32(
 define float @image_atomic_smin(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32.v8i32(
 define float @image_atomic_umin(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32.v8i32(
 define float @image_atomic_smax(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32.v8i32(
 define float @image_atomic_umax(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32.v8i32(
 define float @image_atomic_and(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32.v8i32(
 define float @image_atomic_or(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32.v8i32(
 define float @image_atomic_xor(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32.v8i32(
 define float @image_atomic_inc(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32.v8i32(
 define float @image_atomic_dec(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32.v8i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(
 define float @image_atomic_cmpswap(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data, i32 inreg %cmp) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %data, i32 %cmp, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32 %data, i32 %cmp, i32 %addr, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(
+;CHECK: DIVERGENT: %orig = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32.v8i32(
 define float @image_atomic_add_2d(<8 x i32> inreg %rsrc, i32 inreg %s, i32 inreg %t, i32 inreg %data) #0 {
 main_body:
-  %orig = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(i32 %data, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
+  %orig = call i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32.v8i32(i32 %data, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
   %r = bitcast i32 %orig to float
   ret float %r
 }
 
-declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32(i32, i32, <8 x i32>, i32, i32) #0
-declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #0
-
-declare i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.add.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.sub.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.smin.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.umin.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.smax.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.umax.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.and.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.or.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.xor.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.inc.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.dec.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #0
+
+declare i32 @llvm.amdgcn.image.atomic.add.2d.i32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #0
 
 attributes #0 = { nounwind }
diff --git a/llvm/test/Analysis/ValueTracking/knownbits-rem-lowbits.ll b/llvm/test/Analysis/ValueTracking/knownbits-rem.ll
similarity index 73%
rename from llvm/test/Analysis/ValueTracking/knownbits-rem-lowbits.ll
rename to llvm/test/Analysis/ValueTracking/knownbits-rem.ll
index 0521c7130055f..0aa340c46bdec 100644
--- a/llvm/test/Analysis/ValueTracking/knownbits-rem-lowbits.ll
+++ b/llvm/test/Analysis/ValueTracking/knownbits-rem.ll
@@ -12,6 +12,17 @@ define i8 @urem_low_bits_know(i8 %xx, i8 %yy) {
   ret i8 %r
 }
 
+define i8 @urem_high_bits_know(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @urem_high_bits_know(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = and i8 %xx, 2
+  %y = and i8 %yy, -4
+  %rem = urem i8 %x, %y
+  %r = and i8 %rem, 8
+  ret i8 %r
+}
+
 define i8 @urem_low_bits_know2(i8 %xx, i8 %yy) {
 ; CHECK-LABEL: @urem_low_bits_know2(
 ; CHECK-NEXT:    ret i8 2
@@ -91,6 +102,62 @@ define i8 @srem_low_bits_know2(i8 %xx, i8 %yy) {
   ret i8 %r
 }
 
+define i8 @srem_high_bits_know(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @srem_high_bits_know(
+; CHECK-NEXT:    ret i8 -2
+;
+  %x = or i8 %xx, -2
+  %y = and i8 %yy, -4
+  %rem = srem i8 %x, %y
+  %r = and i8 %rem, -2
+  ret i8 %r
+}
+
+define i8 @srem_high_bits_know2(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @srem_high_bits_know2(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = and i8 %xx, 13
+  %y = or i8 %yy, -4
+  %rem = srem i8 %x, %y
+  %r = and i8 %rem, 8
+  ret i8 %r
+}
+
+define i8 @srem_high_bits_know3(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @srem_high_bits_know3(
+; CHECK-NEXT:    ret i8 8
+;
+  %x = or i8 %xx, -13
+  %y = and i8 %yy, 4
+  %rem = srem i8 %x, %y
+  %r = and i8 %rem, 8
+  ret i8 %r
+}
+
+define i8 @srem_high_bits_know4(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @srem_high_bits_know4(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = and i8 %xx, 4
+  %y = or i8 %yy, -13
+  %rem = srem i8 %x, %y
+  %r = and i8 %rem, 8
+  ret i8 %r
+}
+
+define i8 @srem_high_bits_know5(i8 %xx, i8 %yy) {
+; CHECK-LABEL: @srem_high_bits_know5(
+; CHECK-NEXT:    [[X:%.*]] = and i8 [[XX:%.*]], 2
+; CHECK-NEXT:    ret i8 [[X]]
+;
+  %x = and i8 %xx, 2
+  %y = and i8 %yy, 4
+  %rem = srem i8 %x, %y
+  %r = and i8 %rem, 2
+  ret i8 %r
+}
+
 define i8 @srem_todo_low_bits_partially_know_should_fold_out_srem(i8 %xx, i8 %yy) {
 ; CHECK-LABEL: @srem_todo_low_bits_partially_know_should_fold_out_srem(
 ; CHECK-NEXT:    [[X:%.*]] = or i8 [[XX:%.*]], 10
diff --git a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
index 7e4a4d527fc90..43ac246055da7 100644
--- a/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_nvvm_intrinsics.ll
@@ -31,6 +31,10 @@ declare float @llvm.nvvm.bitcast.i2f(i32)
 declare i64 @llvm.nvvm.bitcast.d2ll(double)
 declare double @llvm.nvvm.bitcast.ll2d(i64)
 
+declare i32 @llvm.nvvm.rotate.b32(i32, i32)
+declare i64 @llvm.nvvm.rotate.right.b64(i64, i32)
+declare i64 @llvm.nvvm.rotate.b64(i64, i32)
+
 ; CHECK-LABEL: @simple_upgrade
 define void @simple_upgrade(i32 %a, i64 %b, i16 %c) {
 ; CHECK: call i32 @llvm.bitreverse.i32(i32 %a)
@@ -139,4 +143,16 @@ define void @bitcast(i32 %a, i64 %b, float %c, double %d) {
   %r4 = call double @llvm.nvvm.bitcast.ll2d(i64 %b)
 
   ret void
-}
\ No newline at end of file
+}
+
+; CHECK-LABEL: @rotate
+define void @rotate(i32 %a, i64 %b) {
+; CHECK: call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 6)
+; CHECK: call i64 @llvm.fshr.i64(i64 %b, i64 %b, i64 7)
+; CHECK: call i64 @llvm.fshl.i64(i64 %b, i64 %b, i64 8)
+;
+  %r1 = call i32 @llvm.nvvm.rotate.b32(i32 %a, i32 6)
+  %r2 = call i64 @llvm.nvvm.rotate.right.b64(i64 %b, i32 7)
+  %r3 = call i64 @llvm.nvvm.rotate.b64(i64 %b, i32 8)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith-fp16.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith-fp16.mir
index f69cbd73b4b32..42538d58c87bf 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith-fp16.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith-fp16.mir
@@ -12,21 +12,24 @@ body:             |
 
     ; NO-FP16-LABEL: name: fadd
     ; NO-FP16: liveins: $h0, $h1
-    ; NO-FP16: %x:_(s16) = COPY $h0
-    ; NO-FP16: %y:_(s16) = COPY $h1
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
-    ; NO-FP16: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
-    ; NO-FP16: %op:_(s16) = G_FPTRUNC [[FADD]](s32)
-    ; NO-FP16: $h0 = COPY %op(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: %x:_(s16) = COPY $h0
+    ; NO-FP16-NEXT: %y:_(s16) = COPY $h1
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
+    ; NO-FP16-NEXT: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
+    ; NO-FP16-NEXT: %op:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; NO-FP16-NEXT: $h0 = COPY %op(s16)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
     ; FP16-LABEL: name: fadd
     ; FP16: liveins: $h0, $h1
-    ; FP16: %x:_(s16) = COPY $h0
-    ; FP16: %y:_(s16) = COPY $h1
-    ; FP16: %op:_(s16) = G_FADD %x, %y
-    ; FP16: $h0 = COPY %op(s16)
-    ; FP16: RET_ReallyLR implicit $h0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: %x:_(s16) = COPY $h0
+    ; FP16-NEXT: %y:_(s16) = COPY $h1
+    ; FP16-NEXT: %op:_(s16) = G_FADD %x, %y
+    ; FP16-NEXT: $h0 = COPY %op(s16)
+    ; FP16-NEXT: RET_ReallyLR implicit $h0
     %x:_(s16) = COPY $h0
     %y:_(s16) = COPY $h1
     %op:_(s16) = G_FADD %x, %y
@@ -43,21 +46,24 @@ body:             |
 
     ; NO-FP16-LABEL: name: fsub
     ; NO-FP16: liveins: $h0, $h1
-    ; NO-FP16: %x:_(s16) = COPY $h0
-    ; NO-FP16: %y:_(s16) = COPY $h1
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
-    ; NO-FP16: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[FPEXT]], [[FPEXT1]]
-    ; NO-FP16: %op:_(s16) = G_FPTRUNC [[FSUB]](s32)
-    ; NO-FP16: $h0 = COPY %op(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: %x:_(s16) = COPY $h0
+    ; NO-FP16-NEXT: %y:_(s16) = COPY $h1
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
+    ; NO-FP16-NEXT: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[FPEXT]], [[FPEXT1]]
+    ; NO-FP16-NEXT: %op:_(s16) = G_FPTRUNC [[FSUB]](s32)
+    ; NO-FP16-NEXT: $h0 = COPY %op(s16)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
     ; FP16-LABEL: name: fsub
     ; FP16: liveins: $h0, $h1
-    ; FP16: %x:_(s16) = COPY $h0
-    ; FP16: %y:_(s16) = COPY $h1
-    ; FP16: %op:_(s16) = G_FSUB %x, %y
-    ; FP16: $h0 = COPY %op(s16)
-    ; FP16: RET_ReallyLR implicit $h0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: %x:_(s16) = COPY $h0
+    ; FP16-NEXT: %y:_(s16) = COPY $h1
+    ; FP16-NEXT: %op:_(s16) = G_FSUB %x, %y
+    ; FP16-NEXT: $h0 = COPY %op(s16)
+    ; FP16-NEXT: RET_ReallyLR implicit $h0
     %x:_(s16) = COPY $h0
     %y:_(s16) = COPY $h1
     %op:_(s16) = G_FSUB %x, %y
@@ -74,21 +80,24 @@ body:             |
 
     ; NO-FP16-LABEL: name: fmul
     ; NO-FP16: liveins: $h0, $h1
-    ; NO-FP16: %x:_(s16) = COPY $h0
-    ; NO-FP16: %y:_(s16) = COPY $h1
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
-    ; NO-FP16: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
-    ; NO-FP16: %op:_(s16) = G_FPTRUNC [[FMUL]](s32)
-    ; NO-FP16: $h0 = COPY %op(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: %x:_(s16) = COPY $h0
+    ; NO-FP16-NEXT: %y:_(s16) = COPY $h1
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
+    ; NO-FP16-NEXT: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
+    ; NO-FP16-NEXT: %op:_(s16) = G_FPTRUNC [[FMUL]](s32)
+    ; NO-FP16-NEXT: $h0 = COPY %op(s16)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
     ; FP16-LABEL: name: fmul
     ; FP16: liveins: $h0, $h1
-    ; FP16: %x:_(s16) = COPY $h0
-    ; FP16: %y:_(s16) = COPY $h1
-    ; FP16: %op:_(s16) = G_FMUL %x, %y
-    ; FP16: $h0 = COPY %op(s16)
-    ; FP16: RET_ReallyLR implicit $h0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: %x:_(s16) = COPY $h0
+    ; FP16-NEXT: %y:_(s16) = COPY $h1
+    ; FP16-NEXT: %op:_(s16) = G_FMUL %x, %y
+    ; FP16-NEXT: $h0 = COPY %op(s16)
+    ; FP16-NEXT: RET_ReallyLR implicit $h0
     %x:_(s16) = COPY $h0
     %y:_(s16) = COPY $h1
     %op:_(s16) = G_FMUL %x, %y
@@ -105,21 +114,24 @@ body:             |
 
     ; NO-FP16-LABEL: name: fdiv
     ; NO-FP16: liveins: $h0, $h1
-    ; NO-FP16: %x:_(s16) = COPY $h0
-    ; NO-FP16: %y:_(s16) = COPY $h1
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
-    ; NO-FP16: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
-    ; NO-FP16: [[FDIV:%[0-9]+]]:_(s32) = G_FDIV [[FPEXT]], [[FPEXT1]]
-    ; NO-FP16: %op:_(s16) = G_FPTRUNC [[FDIV]](s32)
-    ; NO-FP16: $h0 = COPY %op(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: %x:_(s16) = COPY $h0
+    ; NO-FP16-NEXT: %y:_(s16) = COPY $h1
+    ; NO-FP16-NEXT: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
+    ; NO-FP16-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT %y(s16)
+    ; NO-FP16-NEXT: [[FDIV:%[0-9]+]]:_(s32) = G_FDIV [[FPEXT]], [[FPEXT1]]
+    ; NO-FP16-NEXT: %op:_(s16) = G_FPTRUNC [[FDIV]](s32)
+    ; NO-FP16-NEXT: $h0 = COPY %op(s16)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
     ; FP16-LABEL: name: fdiv
     ; FP16: liveins: $h0, $h1
-    ; FP16: %x:_(s16) = COPY $h0
-    ; FP16: %y:_(s16) = COPY $h1
-    ; FP16: %op:_(s16) = G_FDIV %x, %y
-    ; FP16: $h0 = COPY %op(s16)
-    ; FP16: RET_ReallyLR implicit $h0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: %x:_(s16) = COPY $h0
+    ; FP16-NEXT: %y:_(s16) = COPY $h1
+    ; FP16-NEXT: %op:_(s16) = G_FDIV %x, %y
+    ; FP16-NEXT: $h0 = COPY %op(s16)
+    ; FP16-NEXT: RET_ReallyLR implicit $h0
     %x:_(s16) = COPY $h0
     %y:_(s16) = COPY $h1
     %op:_(s16) = G_FDIV %x, %y
@@ -136,18 +148,22 @@ body:             |
 
     ; NO-FP16-LABEL: name: fneg
     ; NO-FP16: liveins: $h0
-    ; NO-FP16: %x:_(s16) = COPY $h0
-    ; NO-FP16: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %x(s16)
-    ; NO-FP16: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
-    ; NO-FP16: %op:_(s16) = G_FPTRUNC [[FNEG]](s32)
-    ; NO-FP16: $h0 = COPY %op(s16)
-    ; NO-FP16: RET_ReallyLR implicit $h0
+    ; NO-FP16-NEXT: {{  $}}
+    ; NO-FP16-NEXT: %x:_(s16) = COPY $h0
+    ; NO-FP16-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT %x(s16)
+    ; NO-FP16-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -32768
+    ; NO-FP16-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT]], [[C]]
+    ; NO-FP16-NEXT: %op:_(s16) = G_TRUNC [[XOR]](s32)
+    ; NO-FP16-NEXT: $h0 = COPY %op(s16)
+    ; NO-FP16-NEXT: RET_ReallyLR implicit $h0
+    ;
     ; FP16-LABEL: name: fneg
     ; FP16: liveins: $h0
-    ; FP16: %x:_(s16) = COPY $h0
-    ; FP16: %op:_(s16) = G_FNEG %x
-    ; FP16: $h0 = COPY %op(s16)
-    ; FP16: RET_ReallyLR implicit $h0
+    ; FP16-NEXT: {{  $}}
+    ; FP16-NEXT: %x:_(s16) = COPY $h0
+    ; FP16-NEXT: %op:_(s16) = G_FNEG %x
+    ; FP16-NEXT: $h0 = COPY %op(s16)
+    ; FP16-NEXT: RET_ReallyLR implicit $h0
     %x:_(s16) = COPY $h0
     %op:_(s16) = G_FNEG %x
     $h0 = COPY %op(s16)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index db2412de48b56..a21b786a2bae9 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -717,6 +717,9 @@
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: G_FATAN2 (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined 
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_FCOSH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index 8710703ab970e..e058c83f274f1 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -1,500 +1,817 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK-CVT --check-prefix=CHECK-COMMON
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-FP16
-
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple \
-; RUN: -asm-verbose=false -disable-post-ra -frame-pointer=non-leaf -global-isel \
-; RUN: -global-isel-abort=2 -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
-; RUN: --check-prefixes=FALLBACK,GISEL-CVT,GISEL
-
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 \
-; RUN: -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra \
-; RUN: -frame-pointer=non-leaf -global-isel -global-isel-abort=2 \
-; RUN: -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
-; RUN: --check-prefixes=FALLBACK-FP16,GISEL-FP16,GISEL
-
-target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-
-; CHECK-CVT-LABEL: test_fadd:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fadd:
-; CHECK-FP16-NEXT:  fadd h0, h0, h1
-; CHECK-FP16-NEXT:  ret
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple aarch64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-SD
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-SD
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-GI
 
 define half @test_fadd(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fadd:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fadd:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fadd:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = fadd half %a, %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fsub:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fsub s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fsub:
-; CHECK-FP16-NEXT: fsub h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
 define half @test_fsub(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fsub:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fsub s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fsub:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fsub h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fsub:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fsub s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = fsub half %a, %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fmul:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fmul:
-; CHECK-FP16-NEXT: fmul h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
 define half @test_fmul(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fmul:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fmul:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fmul h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fmul:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = fmul half %a, %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fmadd:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvt s1, h2
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fmadd:
-; CHECK-FP16-NEXT: fmadd h0, h0, h1, h2
-; CHECK-FP16-NEXT: ret
-
 define half @test_fmadd(half %a, half %b, half %c) #0 {
+; CHECK-CVT-SD-LABEL: test_fmadd:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt s1, h2
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fmadd:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fmadd:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt s1, h2
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %mul = fmul fast half %a, %b
   %r = fadd fast half %mul, %c
   ret half %r
 }
-; CHECK-CVT-LABEL: test_fdiv:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fdiv s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fdiv:
-; CHECK-FP16-NEXT: fdiv	h0, h0, h1
-; CHECK-FP16-NEXT: ret
 
 define half @test_fdiv(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fdiv:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fdiv s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fdiv:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fdiv h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fdiv:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fdiv s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = fdiv half %a, %b
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_frem:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: fcvt s1, h1
-; CHECK-COMMON-NEXT: bl {{_?}}fmodf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
 define half @test_frem(half %a, half %b) #0 {
+; CHECK-LABEL: test_frem:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    bl fmodf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = frem half %a, %b
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_store:
-; CHECK-COMMON-NEXT: str  h0, [x0]
-; CHECK-COMMON-NEXT: ret
 define void @test_store(half %a, ptr %b) #0 {
+; CHECK-LABEL: test_store:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str h0, [x0]
+; CHECK-NEXT:    ret
   store half %a, ptr %b
   ret void
 }
 
-; CHECK-COMMON-LABEL: test_load:
-; CHECK-COMMON-NEXT: ldr  h0, [x0]
-; CHECK-COMMON-NEXT: ret
 define half @test_load(ptr %a) #0 {
+; CHECK-LABEL: test_load:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    ret
   %r = load half, ptr %a
   ret half %r
 }
 
 declare half @test_callee(half %a, half %b) #0
 
-; CHECK-COMMON-LABEL: test_call:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: bl {{_?}}test_callee
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
 define half @test_call(half %a, half %b) #0 {
+; CHECK-LABEL: test_call:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl test_callee
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @test_callee(half %a, half %b)
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_call_flipped:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fmov  s2, s0
-; CHECK-COMMON-NEXT: fmov  s0, s1
-; CHECK-COMMON-NEXT: fmov  s1, s2
-; CHECK-COMMON-NEXT: bl {{_?}}test_callee
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
 define half @test_call_flipped(half %a, half %b) #0 {
+; CHECK-LABEL: test_call_flipped:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fmov s2, s0
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    fmov s1, s2
+; CHECK-NEXT:    bl test_callee
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @test_callee(half %b, half %a)
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_tailcall_flipped:
-; CHECK-COMMON-NEXT: fmov  s2, s0
-; CHECK-COMMON-NEXT: fmov  s0, s1
-; CHECK-COMMON-NEXT: fmov  s1, s2
-; CHECK-COMMON-NEXT: b {{_?}}test_callee
 define half @test_tailcall_flipped(half %a, half %b) #0 {
+; CHECK-LABEL: test_tailcall_flipped:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s2, s0
+; CHECK-NEXT:    fmov s0, s1
+; CHECK-NEXT:    fmov s1, s2
+; CHECK-NEXT:    b test_callee
   %r = tail call half @test_callee(half %b, half %a)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_select:
-; CHECK-CVT-NEXT: cmp  w0, #0
-; CHECK-CVT-NEXT: fcsel s0, s0, s1, ne
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_select:
-; CHECK-FP16-NEXT: cmp w0, #0
-; CHECK-FP16-NEXT: fcsel h0, h0, h1, ne
-; CHECK-FP16-NEXT: ret
-
 define half @test_select(half %a, half %b, i1 zeroext %c) #0 {
+; CHECK-CVT-SD-LABEL: test_select:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-SD-NEXT:    cmp w0, #0
+; CHECK-CVT-SD-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-SD-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_select:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    cmp w0, #0
+; CHECK-FP16-SD-NEXT:    fcsel h0, h0, h1, ne
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_select:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-GI-NEXT:    fmov w8, s0
+; CHECK-CVT-GI-NEXT:    fmov w9, s1
+; CHECK-CVT-GI-NEXT:    tst w0, #0x1
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-CVT-GI-NEXT:    fmov s0, w8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_select:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-FP16-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-FP16-GI-NEXT:    fmov w8, s0
+; CHECK-FP16-GI-NEXT:    fmov w9, s1
+; CHECK-FP16-GI-NEXT:    tst w0, #0x1
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-FP16-GI-NEXT:    fmov s0, w8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-FP16-GI-NEXT:    ret
   %r = select i1 %c, half %a, half %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_select_cc:
-; CHECK-CVT-DAG: fcvt s3, h3
-; CHECK-CVT-DAG: fcvt s2, h2
-; CHECK-CVT-DAG: fcmp s2, s3
-; CHECK-CVT-NEXT: fcsel s0, s0, s1, ne
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_select_cc:
-; CHECK-FP16-NEXT: fcmp h2, h3
-; CHECK-FP16-NEXT: fcsel h0, h0, h1, ne
-; CHECK-FP16-NEXT: ret
-
 define half @test_select_cc(half %a, half %b, half %c, half %d) #0 {
+; CHECK-CVT-SD-LABEL: test_select_cc:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s3, h3
+; CHECK-CVT-SD-NEXT:    fcvt s2, h2
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-SD-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-SD-NEXT:    fcmp s2, s3
+; CHECK-CVT-SD-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_select_cc:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcmp h2, h3
+; CHECK-FP16-SD-NEXT:    fcsel h0, h0, h1, ne
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_select_cc:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s2, h2
+; CHECK-CVT-GI-NEXT:    fcvt s3, h3
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-GI-NEXT:    fmov w8, s0
+; CHECK-CVT-GI-NEXT:    fmov w9, s1
+; CHECK-CVT-GI-NEXT:    fcmp s2, s3
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-CVT-GI-NEXT:    fmov s0, w8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_select_cc:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-FP16-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-FP16-GI-NEXT:    fcmp h2, h3
+; CHECK-FP16-GI-NEXT:    fmov w8, s0
+; CHECK-FP16-GI-NEXT:    fmov w9, s1
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-FP16-GI-NEXT:    fmov s0, w8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-FP16-GI-NEXT:    ret
   %cc = fcmp une half %c, %d
   %r = select i1 %cc, half %a, half %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_select_cc_f32_f16:
-; CHECK-CVT-DAG:   fcvt s2, h2
-; CHECK-CVT-DAG:   fcvt s3, h3
-; CHECK-CVT-NEXT:  fcmp s2, s3
-; CHECK-CVT-NEXT:  fcsel s0, s0, s1, ne
-; CHECK-CVT-NEXT:  ret
-
-; CHECK-FP16-LABEL: test_select_cc_f32_f16:
-; CHECK-FP16-NEXT: fcmp	h2, h3
-; CHECK-FP16-NEXT: fcsel	s0, s0, s1, ne
-; CHECK-FP16-NEXT: ret
-
 define float @test_select_cc_f32_f16(float %a, float %b, half %c, half %d) #0 {
+; CHECK-CVT-SD-LABEL: test_select_cc_f32_f16:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s3, h3
+; CHECK-CVT-SD-NEXT:    fcvt s2, h2
+; CHECK-CVT-SD-NEXT:    fcmp s2, s3
+; CHECK-CVT-SD-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_select_cc_f32_f16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h2, h3
+; CHECK-FP16-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_select_cc_f32_f16:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s2, h2
+; CHECK-CVT-GI-NEXT:    fcvt s3, h3
+; CHECK-CVT-GI-NEXT:    fcmp s2, s3
+; CHECK-CVT-GI-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-CVT-GI-NEXT:    ret
   %cc = fcmp une half %c, %d
   %r = select i1 %cc, float %a, float %b
   ret float %r
 }
 
-; CHECK-CVT-LABEL: test_select_cc_f16_f32:
-; CHECK-CVT-DAG:  fcmp s2, s3
-; CHECK-CVT-NEXT: fcsel s0, s0, s1, ne
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_select_cc_f16_f32:
-; CHECK-FP16-NEXT: fcmp	s2, s3
-; CHECK-FP16-NEXT: fcsel h0, h0, h1, ne
-; CHECK-FP16-NEXT: ret
-
 define half @test_select_cc_f16_f32(half %a, half %b, float %c, float %d) #0 {
+; CHECK-CVT-SD-LABEL: test_select_cc_f16_f32:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcmp s2, s3
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-SD-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-SD-NEXT:    fcsel s0, s0, s1, ne
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_select_cc_f16_f32:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcmp s2, s3
+; CHECK-FP16-SD-NEXT:    fcsel h0, h0, h1, ne
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_select_cc_f16_f32:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-CVT-GI-NEXT:    fcmp s2, s3
+; CHECK-CVT-GI-NEXT:    fmov w8, s0
+; CHECK-CVT-GI-NEXT:    fmov w9, s1
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-CVT-GI-NEXT:    fmov s0, w8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_select_cc_f16_f32:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-FP16-GI-NEXT:    // kill: def $h1 killed $h1 def $s1
+; CHECK-FP16-GI-NEXT:    fcmp s2, s3
+; CHECK-FP16-GI-NEXT:    fmov w8, s0
+; CHECK-FP16-GI-NEXT:    fmov w9, s1
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, ne
+; CHECK-FP16-GI-NEXT:    fmov s0, w8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-FP16-GI-NEXT:    ret
   %cc = fcmp une float %c, %d
   %r = select i1 %cc, half %a, half %b
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_une:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, ne
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_une:
-; CHECK-FP16-NEXT: fcmp	h0, h1
-; CHECK-FP16-NEXT: cset w0, ne
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_une(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_une:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, ne
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_une:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, ne
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_une:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, ne
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp une half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ueq:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset [[TRUE:w[0-9]+]], eq
-; CHECK-CVT-NEXT: csinc w0, [[TRUE]], wzr, vc
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ueq:
-; CHECK-FP16-NEXT: fcmp	h0, h1
-; CHECK-FP16-NEXT: cset [[TRUE:w[0-9]+]], eq
-; CHECK-FP16-NEXT: csinc w0, [[TRUE]], wzr, vc
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ueq(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ueq:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w8, eq
+; CHECK-CVT-SD-NEXT:    csinc w0, w8, wzr, vc
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_fcmp_ueq:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcmp h0, h1
+; CHECK-FP16-SD-NEXT:    cset w8, eq
+; CHECK-FP16-SD-NEXT:    csinc w0, w8, wzr, vc
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ueq:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w8, eq
+; CHECK-CVT-GI-NEXT:    cset w9, vs
+; CHECK-CVT-GI-NEXT:    orr w0, w8, w9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_fcmp_ueq:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    fcmp h0, h1
+; CHECK-FP16-GI-NEXT:    cset w8, eq
+; CHECK-FP16-GI-NEXT:    cset w9, vs
+; CHECK-FP16-GI-NEXT:    orr w0, w8, w9
+; CHECK-FP16-GI-NEXT:    ret
   %r = fcmp ueq half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ugt:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, hi
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ugt:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, hi
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ugt(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ugt:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, hi
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ugt:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, hi
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ugt:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, hi
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ugt half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_uge:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, pl
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_uge:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, pl
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_uge(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_uge:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, pl
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_uge:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, pl
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_uge:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, pl
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp uge half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ult:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, lt
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ult:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, lt
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ult(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ult:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, lt
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ult:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, lt
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ult:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, lt
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ult half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ule:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, le
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ule:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, le
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ule(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ule:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, le
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ule:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, le
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ule:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, le
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ule half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_uno:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, vs
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_uno:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, vs
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_uno(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_uno:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, vs
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_uno:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, vs
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_uno:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, vs
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp uno half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_one:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset [[TRUE:w[0-9]+]], mi
-; CHECK-CVT-NEXT: csinc w0, [[TRUE]], wzr, le
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_one:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset [[TRUE:w[0-9]+]], mi
-; CHECK-FP16-NEXT: csinc w0, [[TRUE]], wzr, le
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_one(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_one:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w8, mi
+; CHECK-CVT-SD-NEXT:    csinc w0, w8, wzr, le
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_fcmp_one:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcmp h0, h1
+; CHECK-FP16-SD-NEXT:    cset w8, mi
+; CHECK-FP16-SD-NEXT:    csinc w0, w8, wzr, le
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_one:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w8, mi
+; CHECK-CVT-GI-NEXT:    cset w9, gt
+; CHECK-CVT-GI-NEXT:    orr w0, w8, w9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_fcmp_one:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    fcmp h0, h1
+; CHECK-FP16-GI-NEXT:    cset w8, mi
+; CHECK-FP16-GI-NEXT:    cset w9, gt
+; CHECK-FP16-GI-NEXT:    orr w0, w8, w9
+; CHECK-FP16-GI-NEXT:    ret
   %r = fcmp one half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_oeq:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, eq
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_oeq:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, eq
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_oeq(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_oeq:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, eq
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_oeq:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, eq
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_oeq:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, eq
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp oeq half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ogt:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, gt
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ogt:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, gt
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ogt(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ogt:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, gt
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ogt:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, gt
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ogt:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, gt
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ogt half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_oge:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, ge
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_oge:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, ge
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_oge(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_oge:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, ge
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_oge:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, ge
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_oge:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, ge
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp oge half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_olt:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, mi
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_olt:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, mi
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_olt(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_olt:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, mi
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_olt:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, mi
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_olt:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, mi
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp olt half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ole:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, ls
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ole:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, ls
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ole(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ole:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, ls
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ole:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, ls
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ole:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, ls
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ole half %a, %b
   ret i1 %r
 }
 
-; CHECK-CVT-LABEL: test_fcmp_ord:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: cset  w0, vc
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fcmp_ord:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: cset  w0, vc
-; CHECK-FP16-NEXT: ret
-
 define i1 @test_fcmp_ord(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_fcmp_ord:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    cset w0, vc
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fcmp_ord:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    cset w0, vc
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fcmp_ord:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    cset w0, vc
+; CHECK-CVT-GI-NEXT:    ret
   %r = fcmp ord half %a, %b
   ret i1 %r
 }
 
-; CHECK-COMMON-LABEL: test_fccmp:
-; CHECK-CVT:      fcvt  s1, h0
-; CHECK-CVT-NEXT: fmov  s2, #5.00000000
-; CHECK-CVT-NEXT: fcmp  s1, s2
-; CHECK-CVT-NEXT: fmov  s2, #8.00000000
-; CHECK-CVT-NEXT: fccmp s1, s2, #4, mi
-; CHECK-CVT-NEXT: adrp x8
-; CHECK-CVT-NEXT: ldr h1, [x8,
-; CHECK-CVT-NEXT: fcsel s0, s0, s1, gt
-; CHECK-CVT-NEXT: str   h0, [x0]
-; CHECK-CVT-NEXT: ret
-; CHECK-FP16:      fmov  h1, #5.00000000
-; CHECK-FP16-NEXT: fcmp  h0, h1
-; CHECK-FP16-NEXT: fmov  h2, #8.00000000
-; CHECK-FP16-NEXT: fccmp h0, h2, #4, mi
-; CHECK-FP16-NEXT: fcsel h0, h0, h1, gt
-; CHECK-FP16-NEXT: str   h0, [x0]
-; CHECK-FP16-NEXT: ret
-
 define void @test_fccmp(half %in, ptr %out) {
+; CHECK-CVT-SD-LABEL: test_fccmp:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-SD-NEXT:    fcvt s1, h0
+; CHECK-CVT-SD-NEXT:    fmov s2, #5.00000000
+; CHECK-CVT-SD-NEXT:    adrp x8, .LCPI29_0
+; CHECK-CVT-SD-NEXT:    fcmp s1, s2
+; CHECK-CVT-SD-NEXT:    fmov s2, #8.00000000
+; CHECK-CVT-SD-NEXT:    fccmp s1, s2, #4, mi
+; CHECK-CVT-SD-NEXT:    ldr h1, [x8, :lo12:.LCPI29_0]
+; CHECK-CVT-SD-NEXT:    fcsel s0, s0, s1, gt
+; CHECK-CVT-SD-NEXT:    str h0, [x0]
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_fccmp:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fmov h1, #5.00000000
+; CHECK-FP16-SD-NEXT:    fmov h2, #8.00000000
+; CHECK-FP16-SD-NEXT:    fcmp h0, h1
+; CHECK-FP16-SD-NEXT:    fccmp h0, h2, #4, mi
+; CHECK-FP16-SD-NEXT:    fcsel h0, h0, h1, gt
+; CHECK-FP16-SD-NEXT:    str h0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fccmp:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    mov w8, #17664 // =0x4500
+; CHECK-CVT-GI-NEXT:    mov w9, #18432 // =0x4800
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-GI-NEXT:    fcvt s2, h0
+; CHECK-CVT-GI-NEXT:    fmov s1, w8
+; CHECK-CVT-GI-NEXT:    fmov s3, w9
+; CHECK-CVT-GI-NEXT:    fmov w9, s0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvt s3, h3
+; CHECK-CVT-GI-NEXT:    fcmp s2, s1
+; CHECK-CVT-GI-NEXT:    fccmp s2, s3, #4, mi
+; CHECK-CVT-GI-NEXT:    csel w8, w9, w8, gt
+; CHECK-CVT-GI-NEXT:    strh w8, [x0]
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_fccmp:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    fmov h1, #5.00000000
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-FP16-GI-NEXT:    fmov h2, #8.00000000
+; CHECK-FP16-GI-NEXT:    fmov w8, s0
+; CHECK-FP16-GI-NEXT:    fcmp h0, h1
+; CHECK-FP16-GI-NEXT:    fmov w9, s1
+; CHECK-FP16-GI-NEXT:    fccmp h0, h2, #4, mi
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, gt
+; CHECK-FP16-GI-NEXT:    strh w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
   %cmp1 = fcmp ogt half %in, 0xH4800
   %cmp2 = fcmp olt half %in, 0xH4500
   %cond = and i1 %cmp1, %cmp2
@@ -503,21 +820,31 @@ define void @test_fccmp(half %in, ptr %out) {
   ret void
 }
 
-; CHECK-CVT-LABEL: test_br_cc:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcmp s0, s1
-; CHECK-CVT-NEXT: csel x8, x0, x1, pl
-; CHECK-CVT-NEXT: str wzr, [x8]
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_br_cc:
-; CHECK-FP16-NEXT: fcmp h0, h1
-; CHECK-FP16-NEXT: csel x8, x0, x1, pl
-; CHECK-FP16-NEXT: str wzr, [x8]
-; CHECK-FP16-NEXT: ret
-
 define void @test_br_cc(half %a, half %b, ptr %p1, ptr %p2) #0 {
+; CHECK-CVT-SD-LABEL: test_br_cc:
+; CHECK-CVT-SD:       // %bb.0: // %common.ret
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcmp s0, s1
+; CHECK-CVT-SD-NEXT:    csel x8, x0, x1, pl
+; CHECK-CVT-SD-NEXT:    str wzr, [x8]
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_br_cc:
+; CHECK-FP16:       // %bb.0: // %common.ret
+; CHECK-FP16-NEXT:    fcmp h0, h1
+; CHECK-FP16-NEXT:    csel x8, x0, x1, pl
+; CHECK-FP16-NEXT:    str wzr, [x8]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_br_cc:
+; CHECK-CVT-GI:       // %bb.0: // %common.ret
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcmp s0, s1
+; CHECK-CVT-GI-NEXT:    csel x8, x0, x1, pl
+; CHECK-CVT-GI-NEXT:    str wzr, [x8]
+; CHECK-CVT-GI-NEXT:    ret
   %c = fcmp uge half %a, %b
   br i1 %c, label %then, label %else
 then:
@@ -528,17 +855,25 @@ else:
   ret void
 }
 
-; CHECK-COMMON-LABEL: test_phi:
-; CHECK-COMMON: mov  x[[PTR:[0-9]+]], x0
-; CHECK-COMMON: ldr  h[[AB:[0-9]+]], [x0]
-; CHECK-COMMON: [[LOOP:LBB[0-9_]+]]:
-; CHECK-COMMON: fmov  s[[R:[0-9]+]], s[[AB]]
-; CHECK-COMMON: ldr  h[[AB]], [x[[PTR]]]
-; CHECK-COMMON: mov  x0, x[[PTR]]
-; CHECK-COMMON: bl {{_?}}test_dummy
-; CHECK-COMMON: fmov  s0, s[[R]]
-; CHECK-COMMON: ret
 define half @test_phi(ptr %p1) #0 {
+; CHECK-LABEL: test_phi:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    ldr h9, [x0]
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:  .LBB31_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    fmov s8, s9
+; CHECK-NEXT:    ldr h9, [x19]
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    bl test_dummy
+; CHECK-NEXT:    tbnz w0, #0, .LBB31_1
+; CHECK-NEXT:  // %bb.2: // %return
+; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    fmov s0, s8
+; CHECK-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
 entry:
   %a = load half, ptr %p1
   br label %loop
@@ -553,208 +888,224 @@ return:
 
 declare i1 @test_dummy(ptr %p1) #0
 
+define i32 @test_fptosi_i32(half %a) #0 {
 ; CHECK-CVT-LABEL: test_fptosi_i32:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvtzs w0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_fptosi_i32:
-; CHECK-FP16-NEXT: fcvtzs w0, h0
-; CHECK-FP16-NEXT: ret
-
-define i32 @test_fptosi_i32(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-NEXT:    ret
   %r = fptosi half %a to i32
   ret i32 %r
 }
 
+define i64 @test_fptosi_i64(half %a) #0 {
 ; CHECK-CVT-LABEL: test_fptosi_i64:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvtzs x0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_fptosi_i64:
-; CHECK-FP16-NEXT: fcvtzs x0, h0
-; CHECK-FP16-NEXT: ret
-
-define i64 @test_fptosi_i64(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs x0, h0
+; CHECK-FP16-NEXT:    ret
   %r = fptosi half %a to i64
   ret i64 %r
 }
 
+define i32 @test_fptoui_i32(half %a) #0 {
 ; CHECK-CVT-LABEL: test_fptoui_i32:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvtzu w0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_fptoui_i32:
-; CHECK-FP16-NEXT: fcvtzu w0, h0
-; CHECK-FP16-NEXT: ret
-
-define i32 @test_fptoui_i32(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu w0, h0
+; CHECK-FP16-NEXT:    ret
   %r = fptoui half %a to i32
   ret i32 %r
 }
 
+define i64 @test_fptoui_i64(half %a) #0 {
 ; CHECK-CVT-LABEL: test_fptoui_i64:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvtzu x0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_fptoui_i64:
-; CHECK-FP16-NEXT: fcvtzu x0, h0
-; CHECK-FP16-NEXT: ret
-
-define i64 @test_fptoui_i64(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu x0, h0
+; CHECK-FP16-NEXT:    ret
   %r = fptoui half %a to i64
   ret i64 %r
 }
 
+define half @test_uitofp_i32(i32 %a) #0 {
 ; CHECK-CVT-LABEL: test_uitofp_i32:
-; CHECK-CVT-NEXT: ucvtf s0, w0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf s0, w0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_uitofp_i32:
-; CHECK-FP16-NEXT: ucvtf h0, w0
-; CHECK-FP16-NEXT: ret
-
-define half @test_uitofp_i32(i32 %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    ucvtf h0, w0
+; CHECK-FP16-NEXT:    ret
   %r = uitofp i32 %a to half
   ret half %r
 }
 
+define half @test_uitofp_i64(i64 %a) #0 {
 ; CHECK-CVT-LABEL: test_uitofp_i64:
-; CHECK-CVT-NEXT: ucvtf s0, x0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf s0, x0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_uitofp_i64:
-; CHECK-FP16-NEXT: ucvtf h0, x0
-; CHECK-FP16-NEXT: ret
-
-define half @test_uitofp_i64(i64 %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    ucvtf h0, x0
+; CHECK-FP16-NEXT:    ret
   %r = uitofp i64 %a to half
   ret half %r
 }
 
+define half @test_sitofp_i32(i32 %a) #0 {
 ; CHECK-CVT-LABEL: test_sitofp_i32:
-; CHECK-CVT-NEXT: scvtf s0, w0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf s0, w0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_sitofp_i32:
-; CHECK-FP16-NEXT: scvtf h0, w0
-; CHECK-FP16-NEXT: ret
-
-define half @test_sitofp_i32(i32 %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    scvtf h0, w0
+; CHECK-FP16-NEXT:    ret
   %r = sitofp i32 %a to half
   ret half %r
 }
 
+define half @test_sitofp_i64(i64 %a) #0 {
 ; CHECK-CVT-LABEL: test_sitofp_i64:
-; CHECK-CVT-NEXT: scvtf s0, x0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf s0, x0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_sitofp_i64:
-; CHECK-FP16-NEXT: scvtf h0, x0
-; CHECK-FP16-NEXT: ret
-define half @test_sitofp_i64(i64 %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    scvtf h0, x0
+; CHECK-FP16-NEXT:    ret
   %r = sitofp i64 %a to half
   ret half %r
 }
 
+define half @test_uitofp_i32_fadd(i32 %a, half %b) #0 {
 ; CHECK-CVT-LABEL: test_uitofp_i32_fadd:
-; CHECK-CVT-NEXT: ucvtf s1, w0
-; CHECK-CVT-NEXT: fcvt h1, s1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ucvtf s1, w0
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvt h1, s1
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_uitofp_i32_fadd:
-; CHECK-FP16-NEXT: ucvtf h1, w0
-; CHECK-FP16-NEXT: fadd h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
-define half @test_uitofp_i32_fadd(i32 %a, half %b) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    ucvtf h1, w0
+; CHECK-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-FP16-NEXT:    ret
   %c = uitofp i32 %a to half
   %r = fadd half %b, %c
   ret half %r
 }
 
+define half @test_sitofp_i32_fadd(i32 %a, half %b) #0 {
 ; CHECK-CVT-LABEL: test_sitofp_i32_fadd:
-; CHECK-CVT-NEXT: scvtf s1, w0
-; CHECK-CVT-NEXT: fcvt h1, s1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    scvtf s1, w0
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvt h1, s1
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_sitofp_i32_fadd:
-; CHECK-FP16-NEXT: scvtf h1, w0
-; CHECK-FP16-NEXT: fadd h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
-define half @test_sitofp_i32_fadd(i32 %a, half %b) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    scvtf h1, w0
+; CHECK-FP16-NEXT:    fadd h0, h0, h1
+; CHECK-FP16-NEXT:    ret
   %c = sitofp i32 %a to half
   %r = fadd half %b, %c
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_fptrunc_float:
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ret
-
 define half @test_fptrunc_float(float %a) #0 {
+; CHECK-LABEL: test_fptrunc_float:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ret
   %r = fptrunc float %a to half
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_fptrunc_double:
-; CHECK-COMMON-NEXT: fcvt h0, d0
-; CHECK-COMMON-NEXT: ret
 define half @test_fptrunc_double(double %a) #0 {
+; CHECK-LABEL: test_fptrunc_double:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt h0, d0
+; CHECK-NEXT:    ret
   %r = fptrunc double %a to half
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_fpext_float:
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: ret
 define float @test_fpext_float(half %a) #0 {
+; CHECK-LABEL: test_fpext_float:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    ret
   %r = fpext half %a to float
   ret float %r
 }
 
-; CHECK-COMMON-LABEL: test_fpext_double:
-; CHECK-COMMON-NEXT: fcvt d0, h0
-; CHECK-COMMON-NEXT: ret
 define double @test_fpext_double(half %a) #0 {
+; CHECK-LABEL: test_fpext_double:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvt d0, h0
+; CHECK-NEXT:    ret
   %r = fpext half %a to double
   ret double %r
 }
 
-
-; CHECK-COMMON-LABEL: test_bitcast_halftoi16:
-; CHECK-COMMON-NEXT: fmov w0, s0
-; CHECK-COMMON-NEXT: ret
 define i16 @test_bitcast_halftoi16(half %a) #0 {
+; CHECK-LABEL: test_bitcast_halftoi16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
   %r = bitcast half %a to i16
   ret i16 %r
 }
 
-; CHECK-COMMON-LABEL: test_bitcast_i16tohalf:
-; CHECK-COMMON-NEXT: fmov s0, w0
-; CHECK-COMMON-NEXT: ret
 define half @test_bitcast_i16tohalf(i16 %a) #0 {
+; CHECK-LABEL: test_bitcast_i16tohalf:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w0
+; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-NEXT:    ret
   %r = bitcast i16 %a to half
   ret half %r
 }
 
-
 declare half @llvm.sqrt.f16(half %a) #0
 declare half @llvm.powi.f16.i32(half %a, i32 %b) #0
 declare half @llvm.sin.f16(half %a) #0
@@ -786,540 +1137,445 @@ declare half @llvm.round.f16(half %a) #0
 declare half @llvm.roundeven.f16(half %a) #0
 declare half @llvm.fmuladd.f16(half %a, half %b, half %c) #0
 
-; FALLBACK-NOT: remark:{{.*}}test_sqrt
-; FALLBACK-FP16-NOT: remark:{{.*}}test_sqrt
 
+define half @test_sqrt(half %a) #0 {
 ; CHECK-CVT-LABEL: test_sqrt:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fsqrt s0, s0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fsqrt s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_sqrt:
-; CHECK-FP16-NEXT: fsqrt h0, h0
-; CHECK-FP16-NEXT: ret
-
-; GISEL-CVT-LABEL: test_sqrt:
-; GISEL-CVT-NEXT: fcvt s0, h0
-; GISEL-CVT-NEXT: fsqrt s0, s0
-; GISEL-CVT-NEXT: fcvt h0, s0
-; GISEL-CVT-NEXT: ret
-
-; GISEL-FP16-LABEL: test_sqrt:
-; GISEL-FP16-NEXT: fsqrt h0, h0
-; GISEL-FP16-NEXT: ret
-
-define half @test_sqrt(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fsqrt h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.sqrt.f16(half %a)
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_powi:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}__powisf2
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
 define half @test_powi(half %a, i32 %b) #0 {
+; CHECK-LABEL: test_powi:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl __powisf2
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.powi.f16.i32(half %a, i32 %b)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_sin
-; FALLBACK-FP16-NOT: remark:{{.*}}test_sin
-
-; CHECK-COMMON-LABEL: test_sin:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}sinf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_sin:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}sinf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
+
 define half @test_sin(half %a) #0 {
+; CHECK-LABEL: test_sin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sinf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.sin.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_cos
-; FALLBACK-FP16-NOT: remark:{{.*}}test_cos
-
-; CHECK-COMMON-LABEL: test_cos:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}cosf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_cos:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}cosf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_cos(half %a) #0 {
+; CHECK-LABEL: test_cos:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl cosf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.cos.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_tan
-; FALLBACK-FP16-NOT: remark:{{.*}}test_tan
-
-; CHECK-COMMON-LABEL: test_tan:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}tanf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_tan:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}tanf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_tan(half %a) #0 {
+; CHECK-LABEL: test_tan:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl tanf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.tan.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_acos
-; FALLBACK-FP16-NOT: remark:{{.*}}test_acos
-
-; CHECK-COMMON-LABEL: test_acos:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}acosf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_acos:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}acosf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_acos(half %a) #0 {
+; CHECK-LABEL: test_acos:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl acosf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.acos.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_asin
-; FALLBACK-FP16-NOT: remark:{{.*}}test_asin
-
-; CHECK-COMMON-LABEL: test_asin:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}asinf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_asin:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}asinf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_asin(half %a) #0 {
+; CHECK-LABEL: test_asin:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl asinf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.asin.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_atan
-; FALLBACK-FP16-NOT: remark:{{.*}}test_atan
-
-; CHECK-COMMON-LABEL: test_atan:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}atanf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_atan:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}atanf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_atan(half %a) #0 {
+; CHECK-LABEL: test_atan:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl atanf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.atan.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_cosh
-; FALLBACK-FP16-NOT: remark:{{.*}}test_cosh
-
-; CHECK-COMMON-LABEL: test_cosh:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}coshf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_cosh:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}coshf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_cosh(half %a) #0 {
+; CHECK-LABEL: test_cosh:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl coshf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.cosh.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_sinh
-; FALLBACK-FP16-NOT: remark:{{.*}}test_sinh
-
-; CHECK-COMMON-LABEL: test_sinh:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}sinhf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_sinh:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}sinhf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_sinh(half %a) #0 {
+; CHECK-LABEL: test_sinh:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl sinhf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.sinh.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_tanh
-; FALLBACK-FP16-NOT: remark:{{.*}}test_tanh
-
-; CHECK-COMMON-LABEL: test_tanh:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}tanhf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_tanh:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}tanhf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_tanh(half %a) #0 {
+; CHECK-LABEL: test_tanh:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl tanhf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.tanh.f16(half %a)
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_pow:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: fcvt s1, h1
-; CHECK-COMMON-NEXT: bl {{_?}}powf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
 define half @test_pow(half %a, half %b) #0 {
+; CHECK-LABEL: test_pow:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    fcvt s1, h1
+; CHECK-NEXT:    bl powf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.pow.f16(half %a, half %b)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_exp
-; FALLBACK-FP16-NOT: remark:{{.*}}test_exp
-
-; CHECK-COMMON-LABEL: test_exp:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}expf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_exp:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}expf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_exp(half %a) #0 {
+; CHECK-LABEL: test_exp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl expf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.exp.f16(half %a)
   ret half %r
 }
 
-; CHECK-COMMON-LABEL: test_exp2:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}exp2f
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_exp2:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}exp2f
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
 define half @test_exp2(half %a) #0 {
+; CHECK-LABEL: test_exp2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl exp2f
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.exp2.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_log
-; FALLBACK-FP16-NOT: remark:{{.*}}test_log
-
-; CHECK-COMMON-LABEL: test_log:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}logf
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_log:
-; GISEL: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}logf
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
-
 define half @test_log(half %a) #0 {
+; CHECK-LABEL: test_log:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl logf
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.log.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_log10
-; FALLBACK-FP16-NOT: remark:{{.*}}test_log10
-
-; CHECK-COMMON-LABEL: test_log10:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}log10f
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_log10:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}log10f
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
-
 define half @test_log10(half %a) #0 {
+; CHECK-LABEL: test_log10:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl log10f
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.log10.f16(half %a)
   ret half %r
 }
 
-; FALLBACK-NOT: remark:{{.*}}test_log2
-; FALLBACK-FP16-NOT: remark:{{.*}}test_log2
-
-; CHECK-COMMON-LABEL: test_log2:
-; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
-; CHECK-COMMON-NEXT: mov  x29, sp
-; CHECK-COMMON-NEXT: fcvt s0, h0
-; CHECK-COMMON-NEXT: bl {{_?}}log2f
-; CHECK-COMMON-NEXT: fcvt h0, s0
-; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
-; CHECK-COMMON-NEXT: ret
-
-; GISEL-LABEL: test_log2:
-; GISEL-NEXT: stp x29, x30, [sp, #-16]!
-; GISEL-NEXT: mov  x29, sp
-; GISEL-NEXT: fcvt s0, h0
-; GISEL-NEXT: bl {{_?}}log2f
-; GISEL-NEXT: fcvt h0, s0
-; GISEL-NEXT: ldp x29, x30, [sp], #16
-; GISEL-NEXT: ret
-
 define half @test_log2(half %a) #0 {
+; CHECK-LABEL: test_log2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    fcvt s0, h0
+; CHECK-NEXT:    bl log2f
+; CHECK-NEXT:    fcvt h0, s0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
   %r = call half @llvm.log2.f16(half %a)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fma:
-; CHECK-CVT-NEXT: fcvt s2, h2
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fmadd s0, s0, s1, s2
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fma:
-; CHECK-FP16-NEXT: fmadd h0, h0, h1, h2
-; CHECK-FP16-NEXT: ret
-
 define half @test_fma(half %a, half %b, half %c) #0 {
+; CHECK-CVT-SD-LABEL: test_fma:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s2, h2
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fma:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fma:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvt s2, h2
+; CHECK-CVT-GI-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = call half @llvm.fma.f16(half %a, half %b, half %c)
   ret half %r
 }
 
+define half @test_fabs(half %a) #0 {
 ; CHECK-CVT-LABEL: test_fabs:
-; CHECK-CVT-NEXT: fmov w8, s0
-; CHECK-CVT-NEXT: and w8, w8, #0x7fff
-; CHECK-CVT-NEXT: fmov s0, w8
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-CVT-NEXT:    fmov w8, s0
+; CHECK-CVT-NEXT:    and w8, w8, #0x7fff
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 killed $s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_fabs:
-; CHECK-FP16-NEXT: fabs h0, h0
-; CHECK-FP16-NEXT: ret
-
-; FALLBACK-NOT: remark:{{.*}}test_fabs
-; FALLBACK-FP16-NOT: remark:{{.*}}test_fabs
-
-; GISEL-CVT-LABEL: test_fabs:
-; GISEL-CVT-NEXT: fcvt s0, h0
-; GISEL-CVT-NEXT: fabs s0, s0
-; GISEL-CVT-NEXT: fcvt h0, s0
-; GISEL-CVT-NEXT: ret
-
-; GISEL-FP16-LABEL: test_fabs:
-; GISEL-FP16-NEXT: fabs h0, h0
-; GISEL-FP16-NEXT: ret
-
-define half @test_fabs(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fabs h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.fabs.f16(half %a)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_minnum:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fminnm s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_minnum:
-; CHECK-FP16-NEXT: fminnm h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
 define half @test_minnum(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_minnum:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fminnm s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_minnum:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fminnm h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_minnum:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fminnm s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = call half @llvm.minnum.f16(half %a, half %b)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_maxnum:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fmaxnm s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_maxnum:
-; CHECK-FP16-NEXT: fmaxnm h0, h0, h1
-; CHECK-FP16-NEXT: ret
-
 define half @test_maxnum(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_maxnum:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fmaxnm s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_maxnum:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fmaxnm h0, h0, h1
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_maxnum:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fmaxnm s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = call half @llvm.maxnum.f16(half %a, half %b)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_copysign:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: mvni.4s v2, #128, lsl #24
-; CHECK-CVT-NEXT: bif.16b v0, v1, v2
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_copysign:
-; CHECK-FP16-NEXT: mvni.8h v2, #128, lsl #8
-; CHECK-FP16-NEXT: bif.16b  v0, v1, v2
-; CHECK-FP16-NEXT: ret
-
 define half @test_copysign(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_copysign:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $h1 killed $h1 def $q1
+; CHECK-FP16-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_copysign:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-GI-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-FP16-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-FP16-GI-NEXT:    ret
   %r = call half @llvm.copysign.f16(half %a, half %b)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_copysign_f32:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: mvni.4s v2, #128, lsl #24
-; CHECK-CVT-NEXT: bif.16b v0, v1, v2
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_copysign_f32:
-; CHECK-FP16-NEXT: fcvt h1, s1
-; CHECK-FP16-NEXT: mvni.8h	v2, #128, lsl #8
-; CHECK-FP16-NEXT: bif.16b v0, v1, v2
-; CHECK-FP16-NEXT: ret
-
 define half @test_copysign_f32(half %a, float %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign_f32:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT:    // kill: def $s1 killed $s1 def $q1
+; CHECK-CVT-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_copysign_f32:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcvt h1, s1
+; CHECK-FP16-SD-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign_f32:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt h1, s1
+; CHECK-CVT-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_copysign_f32:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    fcvt h1, s1
+; CHECK-FP16-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-FP16-GI-NEXT:    ret
   %tb = fptrunc float %b to half
   %r = call half @llvm.copysign.f16(half %a, half %tb)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_copysign_f64:
-; CHECK-CVT-NEXT: fcvt s1, d1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: mvni.4s v2, #128, lsl #24
-; CHECK-CVT-NEXT: bif.16b v0, v1, v2
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_copysign_f64:
-; CHECK-FP16-NEXT: fcvt h1, d1
-; CHECK-FP16-NEXT: mvni.8h v2, #128, lsl #8
-; CHECK-FP16-NEXT: bif.16b v0, v1, v2
-; CHECK-FP16-NEXT: ret
-
 define half @test_copysign_f64(half %a, double %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign_f64:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, d1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_copysign_f64:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    fcvt h1, d1
+; CHECK-FP16-SD-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign_f64:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt h1, d1
+; CHECK-CVT-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_copysign_f64:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    fcvt h1, d1
+; CHECK-FP16-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 killed $d0
+; CHECK-FP16-GI-NEXT:    ret
   %tb = fptrunc double %b to half
   %r = call half @llvm.copysign.f16(half %a, half %tb)
   ret half %r
@@ -1328,204 +1584,190 @@ define half @test_copysign_f64(half %a, double %b) #0 {
 ; Check that the FP promotion will use a truncating FP_ROUND, so we can fold
 ; away the (fpext (fp_round <result>)) here.
 
-; CHECK-CVT-LABEL: test_copysign_extended:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: mvni.4s v2, #128, lsl #24
-; CHECK-CVT-NEXT: bif.16b v0, v1, v2
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_copysign_extended:
-; CHECK-FP16-NEXT: mvni.8h v2, #128, lsl #8
-; CHECK-FP16-NEXT: bif.16b v0, v1, v2
-; CHECK-FP16-NEXT: fcvt s0, h0
-; CHECK-FP16-NEXT: ret
-
 define float @test_copysign_extended(half %a, half %b) #0 {
+; CHECK-CVT-SD-LABEL: test_copysign_extended:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    mvni v2.4s, #128, lsl #24
+; CHECK-CVT-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: test_copysign_extended:
+; CHECK-FP16-SD:       // %bb.0:
+; CHECK-FP16-SD-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $h1 killed $h1 def $q1
+; CHECK-FP16-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    fcvt s0, h0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_copysign_extended:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-CVT-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-GI-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-CVT-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: test_copysign_extended:
+; CHECK-FP16-GI:       // %bb.0:
+; CHECK-FP16-GI-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-GI-NEXT:    // kill: def $h1 killed $h1 def $d1
+; CHECK-FP16-GI-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-FP16-GI-NEXT:    fcvt s0, h0
+; CHECK-FP16-GI-NEXT:    ret
   %r = call half @llvm.copysign.f16(half %a, half %b)
   %xr = fpext half %r to float
   ret float %xr
 }
 
+define half @test_floor(half %a) #0 {
 ; CHECK-CVT-LABEL: test_floor:
-; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; CHECK-CVT-NEXT: frintm [[INT32:s[0-9]+]], [[FLOAT32]]
-; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_floor:
-; CHECK-FP16-NEXT: frintm h0, h0
-; CHECK-FP16-NEXT: ret
-
-; FALLBACK-NOT: remark:{{.*}}test_floor
-; FALLBACK-FP16-NOT: remark:{{.*}}test_floor
-
-; GISEL-CVT-LABEL: test_floor:
-; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; GISEL-CVT-NEXT: frintm [[INT32:s[0-9]+]], [[FLOAT32]]
-; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
-; GISEL-CVT-NEXT: ret
-
-; GISEL-FP16-LABEL: test_floor:
-; GISEL-FP16-NEXT: frintm h0, h0
-; GISEL-FP16-NEXT: ret
-
-define half @test_floor(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frintm h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.floor.f16(half %a)
   ret half %r
 }
 
+define half @test_ceil(half %a) #0 {
 ; CHECK-CVT-LABEL: test_ceil:
-; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; CHECK-CVT-NEXT: frintp [[INT32:s[0-9]+]], [[FLOAT32]]
-; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_ceil:
-; CHECK-FP16-NEXT: frintp h0, h0
-; CHECK-FP16-NEXT: ret
-
-; FALLBACK-NOT: remark:{{.*}}test_ceil
-; FALLBACK-FP16-NOT: remark:{{.*}}test_ceil
-
-; GISEL-CVT-LABEL: test_ceil:
-; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; GISEL-CVT-NEXT: frintp [[INT32:s[0-9]+]], [[FLOAT32]]
-; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
-; GISEL-CVT-NEXT: ret
-
-; GISEL-FP16-LABEL: test_ceil:
-; GISEL-FP16-NEXT: frintp h0, h0
-; GISEL-FP16-NEXT: ret
-define half @test_ceil(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frintp h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.ceil.f16(half %a)
   ret half %r
 }
 
+define half @test_trunc(half %a) #0 {
 ; CHECK-CVT-LABEL: test_trunc:
-; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; CHECK-CVT-NEXT: frintz [[INT32:s[0-9]+]], [[FLOAT32]]
-; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_trunc:
-; CHECK-FP16-NEXT: frintz h0, h0
-; CHECK-FP16-NEXT: ret
-
-define half @test_trunc(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frintz h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.trunc.f16(half %a)
   ret half %r
 }
 
+define half @test_rint(half %a) #0 {
 ; CHECK-CVT-LABEL: test_rint:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: frintx s0, s0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frintx s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_rint:
-; CHECK-FP16-NEXT: frintx h0, h0
-; CHECK-FP16-NEXT: ret
-
-define half @test_rint(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frintx h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.rint.f16(half %a)
   ret half %r
 }
 
+define half @test_nearbyint(half %a) #0 {
 ; CHECK-CVT-LABEL: test_nearbyint:
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: frinti s0, s0
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frinti s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_nearbyint:
-; CHECK-FP16-NEXT: frinti h0, h0
-; CHECK-FP16-NEXT: ret
-
-define half @test_nearbyint(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frinti h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.nearbyint.f16(half %a)
   ret half %r
 }
 
+define half @test_round(half %a) #0 {
 ; CHECK-CVT-LABEL: test_round:
-; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; CHECK-CVT-NEXT: frinta [[INT32:s[0-9]+]], [[FLOAT32]]
-; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
-; CHECK-CVT-NEXT: ret
-
-; GISEL-CVT-LABEL: test_round:
-; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; GISEL-CVT-NEXT: frinta [[INT32:s[0-9]+]], [[FLOAT32]]
-; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
-; GISEL-CVT-NEXT: ret
-
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_round:
-; CHECK-FP16-NEXT: frinta h0, h0
-; CHECK-FP16-NEXT: ret
-
-; GISEL-FP16-LABEL: test_round:
-; GISEL-FP16-NEXT: frinta h0, h0
-; GISEL-FP16-NEXT: ret
-
-define half @test_round(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frinta h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.round.f16(half %a)
   ret half %r
 }
 
+define half @test_roundeven(half %a) #0 {
 ; CHECK-CVT-LABEL: test_roundeven:
-; CHECK-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; CHECK-CVT-NEXT: frintn [[INT32:s[0-9]+]], [[FLOAT32]]
-; CHECK-CVT-NEXT: fcvt h0, [[INT32]]
-; CHECK-CVT-NEXT: ret
-
-; GISEL-CVT-LABEL: test_roundeven:
-; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
-; GISEL-CVT-NEXT: frintn [[INT32:s[0-9]+]], [[FLOAT32]]
-; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
-; GISEL-CVT-NEXT: ret
-
-
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fcvt h0, s0
+; CHECK-CVT-NEXT:    ret
+;
 ; CHECK-FP16-LABEL: test_roundeven:
-; CHECK-FP16-NEXT: frintn h0, h0
-; CHECK-FP16-NEXT: ret
-
-; GISEL-FP16-LABEL: test_roundeven:
-; GISEL-FP16-NEXT: frintn h0, h0
-; GISEL-FP16-NEXT: ret
-
-define half @test_roundeven(half %a) #0 {
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    frintn h0, h0
+; CHECK-FP16-NEXT:    ret
   %r = call half @llvm.roundeven.f16(half %a)
   ret half %r
 }
 
-; CHECK-CVT-LABEL: test_fmuladd:
-; CHECK-CVT-NEXT: fcvt s1, h1
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fmul s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: fcvt s0, h0
-; CHECK-CVT-NEXT: fcvt s1, h2
-; CHECK-CVT-NEXT: fadd s0, s0, s1
-; CHECK-CVT-NEXT: fcvt h0, s0
-; CHECK-CVT-NEXT: ret
-
-; CHECK-FP16-LABEL: test_fmuladd:
-; CHECK-FP16-NEXT: fmadd h0, h0, h1, h2
-; CHECK-FP16-NEXT: ret
-
 define half @test_fmuladd(half %a, half %b, half %c) #0 {
+; CHECK-CVT-SD-LABEL: test_fmuladd:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt s1, h2
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-SD-NEXT:    fcvt h0, s0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_fmuladd:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fmadd h0, h0, h1, h2
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fmuladd:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fmul s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt s1, h2
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fadd s0, s0, s1
+; CHECK-CVT-GI-NEXT:    fcvt h0, s0
+; CHECK-CVT-GI-NEXT:    ret
   %r = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
   ret half %r
 }
 
 attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-COMMON: {{.*}}
-; CHECK-CVT: {{.*}}
-; CHECK-FP16: {{.*}}
-; FALLBACK: {{.*}}
-; FALLBACK-FP16: {{.*}}
-; GISEL: {{.*}}
-; GISEL-CVT: {{.*}}
-; GISEL-FP16: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/fabs.ll b/llvm/test/CodeGen/AArch64/fabs.ll
index e19e2ead11f4d..43e9007073634 100644
--- a/llvm/test/CodeGen/AArch64/fabs.ll
+++ b/llvm/test/CodeGen/AArch64/fabs.ll
@@ -41,9 +41,11 @@ define half @fabs_f16(half %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fabs s0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-GI-NOFP16-NEXT:    fmov w8, s0
+; CHECK-GI-NOFP16-NEXT:    and w8, w8, #0x7fff
+; CHECK-GI-NOFP16-NEXT:    fmov s0, w8
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_f16:
@@ -160,22 +162,8 @@ define <7 x half> @fabs_v7f16(<7 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v7f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[0], v0.h[4]
-; CHECK-GI-NOFP16-NEXT:    fabs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[1], v0.h[5]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[2], v0.h[6]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[0], v1.h[0]
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[1]
-; CHECK-GI-NOFP16-NEXT:    fabs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v1.h[2]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v1.h[3]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v2.h[1]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    mvni v1.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v7f16:
@@ -200,9 +188,8 @@ define <4 x half> @fabs_v4f16(<4 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v4f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fabs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    mvni v1.4h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    and v0.8b, v0.8b, v1.8b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v4f16:
@@ -227,12 +214,8 @@ define <8 x half> @fabs_v8f16(<8 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v8f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fabs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fabs v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    mvni v1.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v8f16:
@@ -259,18 +242,9 @@ define <16 x half> @fabs_v16f16(<16 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v16f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fabs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fabs v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fabs v4.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fabs v5.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    mvni v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT:    and v1.16b, v1.16b, v2.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/fneg.ll b/llvm/test/CodeGen/AArch64/fneg.ll
index a0e9edff733e0..de2671afe60ab 100644
--- a/llvm/test/CodeGen/AArch64/fneg.ll
+++ b/llvm/test/CodeGen/AArch64/fneg.ll
@@ -41,9 +41,11 @@ define half @fabs_f16(half %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fneg s0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 def $s0
+; CHECK-GI-NOFP16-NEXT:    fmov w8, s0
+; CHECK-GI-NOFP16-NEXT:    eor w8, w8, #0xffff8000
+; CHECK-GI-NOFP16-NEXT:    fmov s0, w8
+; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_f16:
@@ -161,22 +163,8 @@ define <7 x half> @fabs_v7f16(<7 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v7f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[0], v0.h[4]
-; CHECK-GI-NOFP16-NEXT:    fneg v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[1], v0.h[5]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    mov v2.h[2], v0.h[6]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[0], v1.h[0]
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[1]
-; CHECK-GI-NOFP16-NEXT:    fneg v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], v1.h[2]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[3], v1.h[3]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[4], v2.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[5], v2.h[1]
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[6], v2.h[2]
+; CHECK-GI-NOFP16-NEXT:    movi v1.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v7f16:
@@ -202,9 +190,8 @@ define <4 x half> @fabs_v4f16(<4 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v4f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fneg v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT:    movi v1.4h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    eor v0.8b, v0.8b, v1.8b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v4f16:
@@ -230,12 +217,8 @@ define <8 x half> @fabs_v8f16(<8 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v8f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fneg v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fneg v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-GI-NOFP16-NEXT:    movi v1.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    eor v0.16b, v0.16b, v1.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v8f16:
@@ -263,18 +246,9 @@ define <16 x half> @fabs_v16f16(<16 x half> %a) {
 ;
 ; CHECK-GI-NOFP16-LABEL: fabs_v16f16:
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fneg v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fneg v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fneg v4.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fneg v5.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-GI-NOFP16-NEXT:    movi v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT:    eor v0.16b, v0.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT:    eor v1.16b, v1.16b, v2.16b
 ; CHECK-GI-NOFP16-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: fabs_v16f16:
diff --git a/llvm/test/CodeGen/AArch64/neon-fp8-fscale.ll b/llvm/test/CodeGen/AArch64/neon-fp8-fscale.ll
new file mode 100644
index 0000000000000..da0e365db2d31
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-fp8-fscale.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=aarch64-linux -mattr=+neon,+fp8 < %s | FileCheck %s
+
+
+define <4 x half> @test_fscale_f16(<4 x half> %vn, <4 x i16> %vm) {
+; CHECK-LABEL: test_fscale_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fscale v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ret
+  %res = tail call <4 x half> @llvm.aarch64.neon.fp8.fscale.v4f16(<4 x half> %vn, <4 x i16> %vm)
+  ret <4 x half> %res
+}
+
+define <8 x half> @test_fscaleq_f16(<8 x half> %vn, <8 x i16> %vm) {
+; CHECK-LABEL: test_fscaleq_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fscale v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    ret
+  %res = tail call <8 x half> @llvm.aarch64.neon.fp8.fscale.v8f16(<8 x half> %vn, <8 x i16> %vm)
+  ret <8 x half> %res
+}
+
+define <2 x float> @test_fscale_f32(<2 x float> %vn, <2 x i32> %vm) {
+; CHECK-LABEL: test_fscale_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fscale v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ret
+  %res = tail call <2 x float> @llvm.aarch64.neon.fp8.fscale.v2f32(<2 x float> %vn, <2 x i32> %vm)
+  ret <2 x float> %res
+}
+
+define <4 x float> @test_fscaleq_f32(<4 x float> %vn, <4 x i32> %vm) {
+; CHECK-LABEL: test_fscaleq_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fscale v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %res = tail call <4 x float> @llvm.aarch64.neon.fp8.fscale.v4f32(<4 x float> %vn, <4 x i32> %vm)
+  ret <4 x float> %res
+}
+
+define <2 x double> @test_fscaleq_f64(<2 x double> %vn, <2 x i64> %vm) {
+; CHECK-LABEL: test_fscaleq_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fscale v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    ret
+  %res = tail call <2 x double> @llvm.aarch64.neon.fp8.fscale.v2f64(<2 x double> %vn, <2 x i64> %vm)
+  ret <2 x double> %res
+}
+
+declare <4 x half> @llvm.aarch64.neon.fp8.fscale.v4f16(<4 x half>, <4 x i16>)
+declare <8 x half> @llvm.aarch64.neon.fp8.fscale.v8f16(<8 x half>, <8 x i16>)
+declare <2 x float> @llvm.aarch64.neon.fp8.fscale.v2f32(<2 x float>, <2 x i32>)
+declare <4 x float> @llvm.aarch64.neon.fp8.fscale.v4f32(<4 x float>, <4 x i32>)
+declare <2 x double> @llvm.aarch64.neon.fp8.fscale.v2f64(<2 x double>, <2 x i64>)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
index 9319f0d3f5d40..c293891140008 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-demote-scc-branches.ll
@@ -359,7 +359,6 @@ if.end:
 
 declare void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32, ptr addrspace(8) nocapture writeonly, i32, i32, i32 immarg)
 declare void @llvm.amdgcn.s.waitcnt(i32)
-declare i32 @llvm.amdgcn.workitem.id.x()
 
 !0 = !{!"branch_weights", i32 1000, i32 1000}
 !1 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
index 5ffdbb0f8c5b0..4df6fcd584ff6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.2dmsaa.ll
@@ -7,10 +7,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -20,7 +20,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(<8 x i32> inreg %rsrc, i3
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0:[0-9]+]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -32,10 +32,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -47,10 +47,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -60,7 +60,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(<8 x i32> inreg %rsrc, i3
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -72,10 +72,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask2(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 2, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 2, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -87,10 +87,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -100,7 +100,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(<8 x i32> inreg %rsrc, i3
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -112,10 +112,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask4(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 4, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 4, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -127,10 +127,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -140,7 +140,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(<8 x i32> inreg %rsrc, i3
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -152,10 +152,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask8(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 8, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 8, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -167,10 +167,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(<8 x i32> inreg %rsrc, i
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -180,7 +180,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(<8 x i32> inreg %rsrc, i
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
@@ -192,10 +192,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_reverse(<8 x i32> inreg %rsrc, i
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -208,10 +208,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(<8 x i32> inreg %rsrc, i32
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S0:%.*]], i32 [[T0:%.*]], i32 [[S1:%.*]], i32 [[T1:%.*]], i32 [[S2:%.*]], i32 [[T2:%.*]], i32 [[S3:%.*]], i32 [[T3:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S0]], i32 [[T0]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S0]], i32 [[T0]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -221,10 +221,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(<8 x i32> inreg %rsrc, i32
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S0:%.*]], i32 [[T0:%.*]], i32 [[S1:%.*]], i32 [[T1:%.*]], i32 [[S2:%.*]], i32 [[T2:%.*]], i32 [[S3:%.*]], i32 [[T3:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S0]], i32 [[T0]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S0]], i32 [[T0]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S1]], i32 [[T1]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S2]], i32 [[T2]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S3]], i32 [[T3]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -232,10 +232,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_vaddr(<8 x i32> inreg %rsrc, i32
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s0, i32 %t0, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s1, i32 %t1, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s2, i32 %t2, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s3, i32 %t3, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s0, i32 %t0, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s1, i32 %t1, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s2, i32 %t2, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s3, i32 %t3, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -247,14 +247,14 @@ define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(<8 x i32> inreg %rsrc, i32 %s, i
 ; NO-MSAA-LABEL: define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I4:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I5:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I6:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I7:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I4:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I5:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I6:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I7:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I8:%.*]] = insertvalue [8 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I9:%.*]] = insertvalue [8 x float] [[I8]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I10:%.*]] = insertvalue [8 x float] [[I9]], float [[I2]], 2
@@ -268,7 +268,7 @@ define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(<8 x i32> inreg %rsrc, i32 %s, i
 ; MSAA-LABEL: define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -288,14 +288,14 @@ define amdgpu_ps [8 x float] @load_2dmsaa_v8f32(<8 x i32> inreg %rsrc, i32 %s, i
 ; MSAA-NEXT:    ret [8 x float] [[I15]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
-  %i4 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i5 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i6 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i7 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i4 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i5 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i6 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i7 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i8 = insertvalue [8 x float] undef, float %i, 0
   %i9 = insertvalue [8 x float] %i8, float %i1, 1
   %i10 = insertvalue [8 x float] %i9, float %i2, 2
@@ -311,20 +311,20 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(<8 x i32> inreg %rsr
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I1:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [4 x float] [[I1]], float [[I2]], 1
-; NO-MSAA-NEXT:    [[I4:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I4:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I3]], float [[I4]], 2
-; NO-MSAA-NEXT:    [[I6:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I6:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I7:%.*]] = insertvalue [4 x float] [[I5]], float [[I6]], 3
 ; NO-MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I4:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -336,13 +336,13 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_interleaved(<8 x i32> inreg %rsr
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   %i1 = insertvalue [4 x float] undef, float %i, 0
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i3 = insertvalue [4 x float] %i1, float %i2, 1
-  %i4 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i4 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
   %i5 = insertvalue [4 x float] %i3, float %i4, 2
-  %i6 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i6 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i7 = insertvalue [4 x float] %i5, float %i6, 3
   ret [4 x float] %i7
 }
@@ -351,8 +351,8 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(<8 x i32> inreg %rsrc,
 ; NO-MSAA-LABEL: define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x float] [[I3]]
@@ -360,7 +360,7 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(<8 x i32> inreg %rsrc,
 ; MSAA-LABEL: define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
@@ -368,8 +368,8 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId01(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = insertvalue [2 x float] undef, float %i, 0
   %i3 = insertvalue [2 x float] %i2, float %i1, 1
   ret [2 x float] %i3
@@ -379,8 +379,8 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(<8 x i32> inreg %rsrc,
 ; NO-MSAA-LABEL: define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x float] [[I2]], float [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x float] [[I3]]
@@ -388,7 +388,7 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(<8 x i32> inreg %rsrc,
 ; MSAA-LABEL: define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 3
 ; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x float] undef, float [[I]], 0
@@ -396,8 +396,8 @@ define amdgpu_ps [2 x float] @load_2dmsaa_v2f32_fragId23(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    ret [2 x float] [[I3]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = insertvalue [2 x float] undef, float %i, 0
   %i3 = insertvalue [2 x float] %i2, float %i1, 1
   ret [2 x float] %i3
@@ -409,8 +409,8 @@ define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [2 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x <2 x float>] [[I5]]
@@ -418,15 +418,15 @@ define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [2 x <2 x float>] @load_2dmsaa_v2v2f32_dmask3(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I4:%.*]] = insertvalue [2 x <2 x float>] undef, <2 x float> [[I]], 0
 ; MSAA-NEXT:    [[I5:%.*]] = insertvalue [2 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; MSAA-NEXT:    ret [2 x <2 x float>] [[I5]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [2 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [2 x <2 x float>] %i4, <2 x float> %i1, 1
   ret [2 x <2 x float>] %i5
@@ -439,10 +439,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -452,8 +452,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -477,10 +477,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask3(<8 x i32> inreg %
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -495,10 +495,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -508,8 +508,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -533,10 +533,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask5(<8 x i32> inreg %
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 5, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 5, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -551,10 +551,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -564,8 +564,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -589,10 +589,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask6(<8 x i32> inreg %
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 6, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 6, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -607,10 +607,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -620,8 +620,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -645,10 +645,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask9(<8 x i32> inreg %
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 9, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 9, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -663,10 +663,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(<8 x i32> inreg
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -676,8 +676,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(<8 x i32> inreg
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -701,10 +701,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask10(<8 x i32> inreg
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 10, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 10, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -719,10 +719,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(<8 x i32> inreg
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -732,8 +732,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(<8 x i32> inreg
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 8, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -757,10 +757,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2dmsaa_v4v2f32_dmask12(<8 x i32> inreg
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32 12, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32 12, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -772,8 +772,8 @@ define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(<8 x i32> inreg %rsrc, i
 ; NO-MSAA-LABEL: define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] undef, half [[I]], 0
 ; NO-MSAA-NEXT:    [[I3:%.*]] = insertvalue [2 x half] [[I2]], half [[I1]], 1
 ; NO-MSAA-NEXT:    ret [2 x half] [[I3]]
@@ -781,7 +781,7 @@ define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(<8 x i32> inreg %rsrc, i
 ; MSAA-LABEL: define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x half> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x half> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f16.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x half> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x half> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = insertvalue [2 x half] undef, half [[I]], 0
@@ -789,8 +789,8 @@ define amdgpu_ps [2 x half] @load_2dmsaa_v2f16_fragId01(<8 x i32> inreg %rsrc, i
 ; MSAA-NEXT:    ret [2 x half] [[I3]]
 ;
 main_body:
-  %i = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   %i2 = insertvalue [2 x half] undef, half %i, 0
   %i3 = insertvalue [2 x half] %i2, half %i1, 1
   ret [2 x half] %i3
@@ -800,10 +800,10 @@ define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(<8 x i32> inreg %rsr
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SLICE:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -813,7 +813,7 @@ define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(<8 x i32> inreg %rsr
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SLICE:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -825,10 +825,10 @@ define amdgpu_ps [4 x float] @load_2darraymsaa_v4f32_dmask1(<8 x i32> inreg %rsr
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -843,10 +843,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(<8 x i32> in
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SLICE:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <2 x float>] undef, <2 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <2 x float>] [[I4]], <2 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <2 x float>] [[I5]], <2 x float> [[I2]], 2
@@ -856,8 +856,8 @@ define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(<8 x i32> in
 ; MSAA-LABEL: define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SLICE:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2darraymsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 [[SLICE]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> undef, float [[TMP2]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -881,10 +881,10 @@ define amdgpu_ps [4 x <2 x float>] @load_2darraymsaa_v4v2f32_dmask3(<8 x i32> in
 ; MSAA-NEXT:    ret [4 x <2 x float>] [[I7]]
 ;
 main_body:
-  %i = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %slice, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <2 x float>] undef, <2 x float> %i, 0
   %i5 = insertvalue [4 x <2 x float>] %i4, <2 x float> %i1, 1
   %i6 = insertvalue [4 x <2 x float>] %i5, <2 x float> %i2, 2
@@ -901,10 +901,10 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(<8 x i32> inreg %
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
@@ -914,9 +914,9 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(<8 x i32> inreg %
 ; MSAA-LABEL: define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> undef, float [[TMP3]], i64 0
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -948,10 +948,10 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7(<8 x i32> inreg %
 ; MSAA-NEXT:    ret [4 x <3 x float>] [[I7]]
 ;
 main_body:
-  %i = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <3 x float>] undef, <3 x float> %i, 0
   %i5 = insertvalue [4 x <3 x float>] %i4, <3 x float> %i1, 1
   %i6 = insertvalue [4 x <3 x float>] %i5, <3 x float> %i2, 2
@@ -968,10 +968,10 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(<8 x i32>
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 5, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 6, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 [[S]], i32 [[T]], i32 7, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 5, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 6, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S]], i32 [[T]], i32 7, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x <3 x float>] undef, <3 x float> [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x <3 x float>] [[I4]], <3 x float> [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x <3 x float>] [[I5]], <3 x float> [[I2]], 2
@@ -981,9 +981,9 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(<8 x i32>
 ; MSAA-LABEL: define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 2, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 4, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 2, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 4, i32 [[S]], i32 [[T]], i32 4, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> undef, float [[TMP3]], i64 0
 ; MSAA-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
@@ -1015,10 +1015,10 @@ define amdgpu_ps [4 x <3 x float>] @load_2dmsaa_v4v3f32_dmask7_group1(<8 x i32>
 ; MSAA-NEXT:    ret [4 x <3 x float>] [[I7]]
 ;
 main_body:
-  %i = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 4, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 5, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 6, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 7, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 4, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 5, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 6, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 7, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x <3 x float>] undef, <3 x float> %i, 0
   %i5 = insertvalue [4 x <3 x float>] %i4, <3 x float> %i1, 1
   %i6 = insertvalue [4 x <3 x float>] %i5, <3 x float> %i2, 2
@@ -1030,11 +1030,11 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(<8 x i32> inreg %rsrc,
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], float [[VDATA:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    call void @llvm.amdgcn.image.store.2dmsaa.f32.i32(float [[VDATA]], i32 1, i32 [[S]], i32 [[T]], i32 [[FRAGID]], <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    call void @llvm.amdgcn.image.store.2dmsaa.f32.i32.v8i32(float [[VDATA]], i32 1, i32 [[S]], i32 [[T]], i32 [[FRAGID]], <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -1044,11 +1044,11 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(<8 x i32> inreg %rsrc,
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], float [[VDATA:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    call void @llvm.amdgcn.image.store.2dmsaa.f32.i32(float [[VDATA]], i32 1, i32 [[S]], i32 [[T]], i32 [[FRAGID]], <8 x i32> [[RSRC]], i32 0, i32 0)
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    call void @llvm.amdgcn.image.store.2dmsaa.f32.i32.v8i32(float [[VDATA]], i32 1, i32 [[S]], i32 [[T]], i32 [[FRAGID]], <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
 ; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
@@ -1058,11 +1058,11 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_sections(<8 x i32> inreg %rsrc,
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
   call void @llvm.amdgcn.image.store.2dmsaa.f32.i32(float %vdata, i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -1074,10 +1074,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -1085,20 +1085,20 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; NO-MSAA-NEXT:    [[I8:%.*]] = trunc i32 [[COND]] to i1
 ; NO-MSAA-NEXT:    br i1 [[I8]], label [[IF_EQUAL:%.*]], label [[IF_UNEQUAL:%.*]]
 ; NO-MSAA:       if_equal:
-; NO-MSAA-NEXT:    [[I9:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I10:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I11:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I12:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I9:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I10:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I11:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I12:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I13:%.*]] = insertvalue [4 x float] undef, float [[I9]], 0
 ; NO-MSAA-NEXT:    [[I14:%.*]] = insertvalue [4 x float] [[I13]], float [[I10]], 1
 ; NO-MSAA-NEXT:    [[I15:%.*]] = insertvalue [4 x float] [[I14]], float [[I11]], 2
 ; NO-MSAA-NEXT:    [[I16:%.*]] = insertvalue [4 x float] [[I15]], float [[I12]], 3
 ; NO-MSAA-NEXT:    br label [[MERGE:%.*]]
 ; NO-MSAA:       if_unequal:
-; NO-MSAA-NEXT:    [[I17:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I18:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I19:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I20:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I17:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I18:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I19:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 2, <8 x i32> [[RSRC]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I20:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 3, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I21:%.*]] = insertvalue [4 x float] undef, float [[I17]], 0
 ; NO-MSAA-NEXT:    [[I22:%.*]] = insertvalue [4 x float] [[I21]], float [[I18]], 1
 ; NO-MSAA-NEXT:    [[I23:%.*]] = insertvalue [4 x float] [[I22]], float [[I19]], 2
@@ -1111,7 +1111,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC:%.*]], i32 [[S:%.*]], i32 [[T:%.*]], i32 [[COND:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP0]], i64 2
@@ -1123,7 +1123,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I8:%.*]] = trunc i32 [[COND]] to i1
 ; MSAA-NEXT:    br i1 [[I8]], label [[IF_EQUAL:%.*]], label [[IF_UNEQUAL:%.*]]
 ; MSAA:       if_equal:
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I9:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I10:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I11:%.*]] = extractelement <4 x float> [[TMP1]], i64 2
@@ -1134,7 +1134,7 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    [[I16:%.*]] = insertvalue [4 x float] [[I15]], float [[I12]], 3
 ; MSAA-NEXT:    br label [[MERGE:%.*]]
 ; MSAA:       if_unequal:
-; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I17:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
 ; MSAA-NEXT:    [[I18:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
 ; MSAA-NEXT:    [[I19:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
@@ -1149,10 +1149,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_blocks(<8 x i32> inreg %rsrc, i3
 ; MSAA-NEXT:    ret [4 x float] [[I25]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -1160,20 +1160,20 @@ main_body:
   %i8 = trunc i32 %cond to i1
   br i1 %i8, label %if_equal, label %if_unequal
 if_equal:
-  %i9 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i10 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i11 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i12 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i9 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i10 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i11 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i12 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i13 = insertvalue [4 x float] undef, float %i9, 0
   %i14 = insertvalue [4 x float] %i13, float %i10, 1
   %i15 = insertvalue [4 x float] %i14, float %i11, 2
   %i16 = insertvalue [4 x float] %i15, float %i12, 3
   br label %merge
 if_unequal:
-  %i17 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
-  %i18 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
-  %i19 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
-  %i20 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
+  %i17 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %i18 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc, i32 0, i32 0)
+  %i19 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 2, <8 x i32> %rsrc, i32 0, i32 0)
+  %i20 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 3, <8 x i32> %rsrc, i32 0, i32 0)
   %i21 = insertvalue [4 x float] undef, float %i17, 0
   %i22 = insertvalue [4 x float] %i21, float %i18, 1
   %i23 = insertvalue [4 x float] %i22, float %i19, 2
@@ -1188,10 +1188,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32>
 ; NO-MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(
 ; NO-MSAA-SAME: <8 x i32> inreg [[RSRC1:%.*]], <8 x i32> inreg [[RSRC2:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; NO-MSAA-NEXT:  main_body:
-; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC1]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
-; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I1:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I2:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; NO-MSAA-NEXT:    [[I3:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 1, <8 x i32> [[RSRC2]], i32 0, i32 0)
 ; NO-MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
 ; NO-MSAA-NEXT:    [[I5:%.*]] = insertvalue [4 x float] [[I4]], float [[I1]], 1
 ; NO-MSAA-NEXT:    [[I6:%.*]] = insertvalue [4 x float] [[I5]], float [[I2]], 2
@@ -1201,10 +1201,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32>
 ; MSAA-LABEL: define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(
 ; MSAA-SAME: <8 x i32> inreg [[RSRC1:%.*]], <8 x i32> inreg [[RSRC2:%.*]], i32 [[S:%.*]], i32 [[T:%.*]]) #[[ATTR0]] {
 ; MSAA-NEXT:  main_body:
-; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC1]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I:%.*]] = extractelement <4 x float> [[TMP0]], i64 0
 ; MSAA-NEXT:    [[I1:%.*]] = extractelement <4 x float> [[TMP0]], i64 1
-; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
+; MSAA-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.2dmsaa.v4f32.i32.v8i32(i32 1, i32 [[S]], i32 [[T]], i32 0, <8 x i32> [[RSRC2]], i32 0, i32 0)
 ; MSAA-NEXT:    [[I2:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
 ; MSAA-NEXT:    [[I3:%.*]] = extractelement <4 x float> [[TMP1]], i64 1
 ; MSAA-NEXT:    [[I4:%.*]] = insertvalue [4 x float] undef, float [[I]], 0
@@ -1214,10 +1214,10 @@ define amdgpu_ps [4 x float] @load_2dmsaa_v4f32_dmask1_different_rsrc(<8 x i32>
 ; MSAA-NEXT:    ret [4 x float] [[I7]]
 ;
 main_body:
-  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc1, i32 0, i32 0)
-  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc1, i32 0, i32 0)
-  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc2, i32 0, i32 0)
-  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc2, i32 0, i32 0)
+  %i = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc1, i32 0, i32 0)
+  %i1 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc1, i32 0, i32 0)
+  %i2 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc2, i32 0, i32 0)
+  %i3 = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 1, <8 x i32> %rsrc2, i32 0, i32 0)
   %i4 = insertvalue [4 x float] undef, float %i, 0
   %i5 = insertvalue [4 x float] %i4, float %i1, 1
   %i6 = insertvalue [4 x float] %i5, float %i2, 2
@@ -1225,16 +1225,16 @@ main_body:
   ret [4 x float] %i7
 }
 
-declare float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare <2 x float> @llvm.amdgcn.image.load.2dmsaa.v2f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare <3 x float> @llvm.amdgcn.image.load.2dmsaa.v3f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
-declare float @llvm.amdgcn.image.load.2darraymsaa.f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare <3 x float> @llvm.amdgcn.image.load.2darraymsaa.v3f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare float @llvm.amdgcn.image.load.2darraymsaa.f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare <2 x float> @llvm.amdgcn.image.load.2darraymsaa.v2f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare <3 x float> @llvm.amdgcn.image.load.2darraymsaa.v3f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
-declare half @llvm.amdgcn.image.load.2dmsaa.f16.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare half @llvm.amdgcn.image.load.2dmsaa.f16.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
-declare void @llvm.amdgcn.image.store.2dmsaa.f32.i32(float, i32, i32, i32, i32, <8 x i32>, i32, i32)
+declare void @llvm.amdgcn.image.store.2dmsaa.f32.i32.v8i32(float, i32, i32, i32, i32, <8 x i32>, i32, i32)
 
 attributes #0 = { nounwind readonly willreturn }
diff --git a/llvm/test/CodeGen/ARM/select-imm.ll b/llvm/test/CodeGen/ARM/select-imm.ll
index 65288e1884c74..6427a3e34cf8e 100644
--- a/llvm/test/CodeGen/ARM/select-imm.ll
+++ b/llvm/test/CodeGen/ARM/select-imm.ll
@@ -655,14 +655,11 @@ define i1 @t10() {
 ; V8MBASE-NEXT:    .pad #8
 ; V8MBASE-NEXT:    sub sp, #8
 ; V8MBASE-NEXT:    movs r0, #7
-; V8MBASE-NEXT:    mvns r0, r0
-; V8MBASE-NEXT:    str r0, [sp]
-; V8MBASE-NEXT:    adds r1, r0, #5
-; V8MBASE-NEXT:    str r1, [sp, #4]
-; V8MBASE-NEXT:    sdiv r2, r1, r0
-; V8MBASE-NEXT:    muls r2, r0, r2
-; V8MBASE-NEXT:    subs r0, r1, r2
-; V8MBASE-NEXT:    subs r1, r0, r1
+; V8MBASE-NEXT:    mvns r1, r0
+; V8MBASE-NEXT:    str r1, [sp]
+; V8MBASE-NEXT:    adds r0, r1, #5
+; V8MBASE-NEXT:    str r0, [sp, #4]
+; V8MBASE-NEXT:    adds r1, #8
 ; V8MBASE-NEXT:    rsbs r0, r1, #0
 ; V8MBASE-NEXT:    adcs r0, r1
 ; V8MBASE-NEXT:    add sp, #8
@@ -719,7 +716,7 @@ define i1 @t11() {
 ; ARMT2-NEXT:    and r1, r1, r2
 ; ARMT2-NEXT:    orr r0, r1, r0
 ; ARMT2-NEXT:    str r0, [sp]
-; ARMT2-NEXT:    bfc r0, #12, #20
+; ARMT2-NEXT:    and r0, r0, #15
 ; ARMT2-NEXT:    sub r0, r0, #3
 ; ARMT2-NEXT:    clz r0, r0
 ; ARMT2-NEXT:    lsr r0, r0, #5
@@ -781,7 +778,7 @@ define i1 @t11() {
 ; THUMB2-NEXT:    ands r1, r2
 ; THUMB2-NEXT:    orrs r0, r1
 ; THUMB2-NEXT:    str r0, [sp]
-; THUMB2-NEXT:    bfc r0, #12, #20
+; THUMB2-NEXT:    and r0, r0, #15
 ; THUMB2-NEXT:    subs r0, #3
 ; THUMB2-NEXT:    clz r0, r0
 ; THUMB2-NEXT:    lsrs r0, r0, #5
diff --git a/llvm/test/CodeGen/ARM/struct_byval.ll b/llvm/test/CodeGen/ARM/struct_byval.ll
index 73a1b5ee33bca..2bc4f9c816d53 100644
--- a/llvm/test/CodeGen/ARM/struct_byval.ll
+++ b/llvm/test/CodeGen/ARM/struct_byval.ll
@@ -63,25 +63,6 @@ declare i32 @e1(ptr nocapture byval(%struct.SmallStruct) %in) nounwind
 declare i32 @e2(ptr nocapture byval(%struct.LargeStruct) %in) nounwind
 declare i32 @e3(ptr nocapture byval(%struct.LargeStruct) align 16 %in) nounwind
 
-; rdar://12442472
-; We can't do tail call since address of s is passed to the callee and part of
-; s is in caller's local frame.
-define void @f3(ptr nocapture byval(%struct.SmallStruct) %s) nounwind optsize {
-; CHECK-LABEL: f3
-; CHECK: bl _consumestruct
-entry:
-  tail call void @consumestruct(ptr %s, i32 80) optsize
-  ret void
-}
-
-define void @f4(ptr nocapture byval(%struct.SmallStruct) %s) nounwind optsize {
-; CHECK-LABEL: f4
-; CHECK: bl _consumestruct
-entry:
-  tail call void @consumestruct(ptr %s, i32 80) optsize
-  ret void
-}
-
 ; We can do tail call here since s is in the incoming argument area.
 define void @f5(i32 %a, i32 %b, i32 %c, i32 %d, ptr nocapture byval(%struct.SmallStruct) %s) nounwind optsize {
 ; CHECK-LABEL: f5
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index 46326d6917587..102748508b4ad 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -8,6 +8,7 @@
 ; CHECK-NEXT: Target Transform Information
 ; CHECK-NEXT: ModulePass Manager
 ; CHECK-NEXT:   DXIL Intrinsic Expansion
+; CHECK-NEXT:   DXIL Data Scalarization
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Dominator Tree Construction
 ; CHECK-NEXT:     Scalarize vector operations
diff --git a/llvm/test/CodeGen/DirectX/scalar-data.ll b/llvm/test/CodeGen/DirectX/scalar-data.ll
new file mode 100644
index 0000000000000..4438604a3a879
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/scalar-data.ll
@@ -0,0 +1,12 @@
+; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
+
+; Make sure we don't touch arrays without vectors and that can recurse multiple-dimension arrays of vectors
+
+@staticArray = internal global [4 x i32] [i32 1, i32 2, i32 3, i32 4], align 4
+@"groushared3dArrayofVectors" = local_unnamed_addr addrspace(3) global [3 x [3 x [3 x <4 x i32>]]] zeroinitializer, align 16
+
+; CHECK @staticArray
+; CHECK-NOT: @staticArray.scalarized
+; CHECK: @groushared3dArrayofVectors.scalarized = local_unnamed_addr addrspace(3) global [3 x [3 x [3 x [4 x i32]]]] zeroinitializer, align 16
+; CHECK-NOT: @groushared3dArrayofVectors
diff --git a/llvm/test/CodeGen/DirectX/scalar-load.ll b/llvm/test/CodeGen/DirectX/scalar-load.ll
new file mode 100644
index 0000000000000..11678f48a5e01
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/scalar-load.ll
@@ -0,0 +1,58 @@
+; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
+
+; Make sure we can load groupshared, static vectors and arrays of vectors
+
+@"arrayofVecData" = local_unnamed_addr addrspace(3) global [2 x <3 x float>] zeroinitializer, align 16
+@"vecData" = external addrspace(3) global <4 x i32>, align 4
+@staticArrayOfVecData = internal global [3 x <4 x i32>] [<4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>, <4 x i32> <i32 9, i32 10, i32 11, i32 12>], align 4
+@"groushared2dArrayofVectors" = local_unnamed_addr addrspace(3) global [3 x [ 3 x <4 x i32>]] zeroinitializer, align 16
+
+; CHECK: @arrayofVecData.scalarized = local_unnamed_addr addrspace(3) global [2 x [3 x float]] zeroinitializer, align 16
+; CHECK: @vecData.scalarized = external addrspace(3) global [4 x i32], align 4
+; CHECK: @staticArrayOfVecData.scalarized = internal global [3 x [4 x i32]] {{\[}}[4 x i32] [i32 1, i32 2, i32 3, i32 4], [4 x i32] [i32 5, i32 6, i32 7, i32 8], [4 x i32] [i32 9, i32 10, i32 11, i32 12]], align 4
+; CHECK: @groushared2dArrayofVectors.scalarized = local_unnamed_addr addrspace(3) global [3 x [3 x [4 x i32]]] zeroinitializer, align 16
+
+; CHECK-NOT: @arrayofVecData
+; CHECK-NOT: @vecData
+; CHECK-NOT: @staticArrayOfVecData
+; CHECK-NOT: @groushared2dArrayofVectors
+
+
+; CHECK-LABEL: load_array_vec_test
+define <4 x i32> @load_array_vec_test() {
+  ; CHECK-COUNT-8: load i32, ptr addrspace(3) {{(.*@arrayofVecData.scalarized.*|%.*)}}, align 4
+  ; CHECK-NOT: load i32, ptr addrspace(3) {{.*}}, align 4
+  %1 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([2 x <4 x i32>], [2 x <4 x i32>] addrspace(3)* @"arrayofVecData", i32 0, i32 0), align 4
+  %2 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([2 x <4 x i32>], [2 x <4 x i32>] addrspace(3)* @"arrayofVecData", i32 0, i32 1), align 4
+  %3 = add <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+; CHECK-LABEL: load_vec_test
+define <4 x i32> @load_vec_test() {
+  ; CHECK-COUNT-4: load i32, ptr addrspace(3) {{(@vecData.scalarized|getelementptr \(i32, ptr addrspace\(3\) @vecData.scalarized, i32 .*\)|%.*)}}, align {{.*}}
+  ; CHECK-NOT: load i32, ptr addrspace(3) {{.*}}, align 4 
+  %1 = load <4 x i32>, <4 x i32> addrspace(3)* @"vecData", align 4
+  ret <4 x i32> %1
+}
+
+; CHECK-LABEL: load_static_array_of_vec_test
+define <4 x i32> @load_static_array_of_vec_test(i32 %index) {
+  ; CHECK: getelementptr [3 x [4 x i32]], ptr @staticArrayOfVecData.scalarized, i32 0, i32 %index
+  ; CHECK-COUNT-4: load i32, ptr {{.*}}, align 4
+  ; CHECK-NOT: load i32, ptr {{.*}}, align 4
+  %3 = getelementptr inbounds [3 x <4 x i32>], [3 x <4 x i32>]* @staticArrayOfVecData, i32 0, i32 %index
+  %4 = load <4 x i32>, <4 x i32>* %3, align 4
+  ret <4 x i32> %4
+}
+
+; CHECK-LABEL: multid_load_test
+define <4 x i32> @multid_load_test()  {
+  ; CHECK-COUNT-8: load i32, ptr addrspace(3) {{(.*@groushared2dArrayofVectors.scalarized.*|%.*)}}, align 4
+  ; CHECK-NOT: load i32, ptr addrspace(3) {{.*}}, align 4
+  %1 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([3 x [3 x <4 x i32>]], [3 x [3 x <4 x i32>]] addrspace(3)* @"groushared2dArrayofVectors", i32 0, i32 0, i32 0), align 4
+  %2 = load <4 x i32>, <4 x i32> addrspace(3)* getelementptr inbounds ([3 x [3 x <4 x i32>]], [3 x [3 x <4 x i32>]] addrspace(3)* @"groushared2dArrayofVectors", i32 0, i32 1, i32 1), align 4
+  %3 = add <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
diff --git a/llvm/test/CodeGen/DirectX/scalar-store.ll b/llvm/test/CodeGen/DirectX/scalar-store.ll
index b970a2842e5a8..08d8a2c57c6c3 100644
--- a/llvm/test/CodeGen/DirectX/scalar-store.ll
+++ b/llvm/test/CodeGen/DirectX/scalar-store.ll
@@ -1,17 +1,29 @@
-; RUN: opt -S -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+; RUN: opt -S -dxil-data-scalarization -scalarizer -scalarize-load-store -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
 ; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
 
-@"sharedData" = local_unnamed_addr addrspace(3) global [2 x <3 x float>] zeroinitializer, align 16 
-; CHECK-LABEL: store_test
-define void @store_test () local_unnamed_addr {
-    ; CHECK: store float 1.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}} 
-    ; CHECK: store float 2.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}}
-    ; CHECK: store float 3.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}} 
-    ; CHECK: store float 2.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}} 
-    ; CHECK: store float 4.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}} 
-    ; CHECK: store float 6.000000e+00, ptr addrspace(3) {{.*}}, align {{.*}} 
+; Make sure we can store groupshared, static vectors and arrays of vectors
 
-    store <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, ptr addrspace(3) @"sharedData", align 16 
-    store <3 x float> <float 2.000000e+00, float 4.000000e+00, float 6.000000e+00>, ptr addrspace(3)   getelementptr inbounds (i8, ptr addrspace(3) @"sharedData", i32 16), align 16 
+@"arrayofVecData" = local_unnamed_addr addrspace(3) global [2 x <3 x float>] zeroinitializer, align 16
+@"vecData" = external addrspace(3) global <4 x i32>, align 4
+
+; CHECK: @arrayofVecData.scalarized = local_unnamed_addr addrspace(3) global [2 x [3 x float]] zeroinitializer, align 16
+; CHECK: @vecData.scalarized = external addrspace(3) global [4 x i32], align 4
+; CHECK-NOT: @arrayofVecData
+; CHECK-NOT: @vecData
+
+; CHECK-LABEL: store_array_vec_test
+define void @store_array_vec_test () local_unnamed_addr {
+    ; CHECK-COUNT-6: store float {{1|2|3|4|6}}.000000e+00, ptr addrspace(3) {{(.*@arrayofVecData.scalarized.*|%.*)}}, align {{4|8|16}}
+    ; CHECK-NOT: store float {{1|2|3|4|6}}.000000e+00, ptr addrspace(3) {{(.*@arrayofVecData.scalarized.*|%.*)}}, align {{4|8|16}}
+    store <3 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, ptr addrspace(3) @"arrayofVecData", align 16 
+    store <3 x float> <float 2.000000e+00, float 4.000000e+00, float 6.000000e+00>, ptr addrspace(3)   getelementptr inbounds (i8, ptr addrspace(3) @"arrayofVecData", i32 16), align 16 
     ret void
  } 
+
+; CHECK-LABEL: store_vec_test
+define void @store_vec_test(<4 x i32> %inputVec) {
+  ; CHECK-COUNT-4: store i32 %inputVec.{{.*}}, ptr addrspace(3) {{(@vecData.scalarized|getelementptr \(i32, ptr addrspace\(3\) @vecData.scalarized, i32 .*\)|%.*)}}, align 4 
+  ; CHECK-NOT: store i32 %inputVec.{{.*}}, ptr addrspace(3)
+  store <4 x i32> %inputVec, <4 x i32> addrspace(3)* @"vecData", align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/NVPTX/cluster-dim.ll b/llvm/test/CodeGen/NVPTX/cluster-dim.ll
new file mode 100644
index 0000000000000..42dff68fa6594
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cluster-dim.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 | FileCheck -check-prefixes=CHECK80 %s
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 | FileCheck -check-prefixes=CHECK90 %s
+; RUN: %if ptxas-12.0 %{ llc < %s -march=nvptx64 -mcpu=sm_90 | %ptxas-verify -arch=sm_90 %}
+
+define void @kernel_func_clusterxyz() {
+; CHECK80-LABEL: kernel_func_clusterxyz(
+; CHECK80:       {
+; CHECK80-EMPTY:
+; CHECK80-EMPTY:
+; CHECK80-NEXT:  // %bb.0:
+; CHECK80-NEXT:    ret;
+;
+; CHECK90-LABEL: kernel_func_clusterxyz(
+; CHECK90:       .explicitcluster
+; CHECK90-NEXT:  .reqnctapercluster 3, 5, 7
+; CHECK90-NEXT:  {
+; CHECK90-EMPTY:
+; CHECK90-EMPTY:
+; CHECK90-NEXT:  // %bb.0:
+; CHECK90-NEXT:    ret;
+  ret void
+}
+
+
+!nvvm.annotations = !{!1, !2}
+
+!1 = !{ptr @kernel_func_clusterxyz, !"kernel", i32 1}
+!2 = !{ptr @kernel_func_clusterxyz, !"cluster_dim_x", i32 3, !"cluster_dim_y", i32 5, !"cluster_dim_z", i32 7}
diff --git a/llvm/test/CodeGen/NVPTX/rotate.ll b/llvm/test/CodeGen/NVPTX/rotate.ll
index 20c7ae5908d29..6586393f83d44 100644
--- a/llvm/test/CodeGen/NVPTX/rotate.ll
+++ b/llvm/test/CodeGen/NVPTX/rotate.ll
@@ -9,26 +9,29 @@ declare i32 @llvm.nvvm.rotate.b32(i32, i32)
 declare i64 @llvm.nvvm.rotate.b64(i64, i32)
 declare i64 @llvm.nvvm.rotate.right.b64(i64, i32)
 
+declare i64 @llvm.fshl.i64(i64, i64, i64)
+declare i64 @llvm.fshr.i64(i64, i64, i64)
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+
 ; SM20: rotate32
 ; SM35: rotate32
 define i32 @rotate32(i32 %a, i32 %b) {
 ; SM20-LABEL: rotate32(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<4>;
+; SM20-NEXT:    .reg .b32 %r<9>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u32 %r1, [rotate32_param_0];
 ; SM20-NEXT:    ld.param.u32 %r2, [rotate32_param_1];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b32 %lhs;
-; SM20-NEXT:    .reg .b32 %rhs;
-; SM20-NEXT:    .reg .b32 %amt2;
-; SM20-NEXT:    shl.b32 %lhs, %r1, %r2;
-; SM20-NEXT:    sub.s32 %amt2, 32, %r2;
-; SM20-NEXT:    shr.b32 %rhs, %r1, %amt2;
-; SM20-NEXT:    add.u32 %r3, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM20-NEXT:    and.b32 %r3, %r2, 31;
+; SM20-NEXT:    shl.b32 %r4, %r1, %r3;
+; SM20-NEXT:    neg.s32 %r5, %r2;
+; SM20-NEXT:    and.b32 %r6, %r5, 31;
+; SM20-NEXT:    shr.u32 %r7, %r1, %r6;
+; SM20-NEXT:    or.b32 %r8, %r4, %r7;
+; SM20-NEXT:    st.param.b32 [func_retval0+0], %r8;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotate32(
@@ -50,45 +53,36 @@ define i32 @rotate32(i32 %a, i32 %b) {
 define i64 @rotate64(i64 %a, i32 %b) {
 ; SM20-LABEL: rotate64(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<2>;
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotate64_param_0];
 ; SM20-NEXT:    ld.param.u32 %r1, [rotate64_param_1];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    .reg .u32 %amt2;
-; SM20-NEXT:    and.b32 %amt2, %r1, 63;
-; SM20-NEXT:    shl.b64 %lhs, %rd1, %amt2;
-; SM20-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM20-NEXT:    shr.b64 %rhs, %rd1, %amt2;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM20-NEXT:    neg.s32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shr.u64 %rd3, %rd1, %r4;
+; SM20-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotate64(
 ; SM35:       {
-; SM35-NEXT:    .reg .b32 %r<6>;
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotate64_param_0];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b32 %dummy;
-; SM35-NEXT:    mov.b64 {%dummy,%r1}, %rd1;
-; SM35-NEXT:    }
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b32 %dummy;
-; SM35-NEXT:    mov.b64 {%r2,%dummy}, %rd1;
-; SM35-NEXT:    }
-; SM35-NEXT:    ld.param.u32 %r3, [rotate64_param_1];
-; SM35-NEXT:    shf.l.wrap.b32 %r4, %r2, %r1, %r3;
-; SM35-NEXT:    shf.l.wrap.b32 %r5, %r1, %r2, %r3;
-; SM35-NEXT:    mov.b64 %rd2, {%r5, %r4};
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    ld.param.u32 %r1, [rotate64_param_1];
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM35-NEXT:    neg.s32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shr.u64 %rd3, %rd1, %r4;
+; SM35-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.nvvm.rotate.b64(i64 %a, i32 %b)
   ret i64 %val
@@ -99,45 +93,36 @@ define i64 @rotate64(i64 %a, i32 %b) {
 define i64 @rotateright64(i64 %a, i32 %b) {
 ; SM20-LABEL: rotateright64(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<2>;
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotateright64_param_0];
 ; SM20-NEXT:    ld.param.u32 %r1, [rotateright64_param_1];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    .reg .u32 %amt2;
-; SM20-NEXT:    and.b32 %amt2, %r1, 63;
-; SM20-NEXT:    shr.b64 %lhs, %rd1, %amt2;
-; SM20-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM20-NEXT:    shl.b64 %rhs, %rd1, %amt2;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    shr.u64 %rd2, %rd1, %r2;
+; SM20-NEXT:    neg.s32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shl.b64 %rd3, %rd1, %r4;
+; SM20-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotateright64(
 ; SM35:       {
-; SM35-NEXT:    .reg .b32 %r<6>;
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotateright64_param_0];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b32 %dummy;
-; SM35-NEXT:    mov.b64 {%r1,%dummy}, %rd1;
-; SM35-NEXT:    }
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b32 %dummy;
-; SM35-NEXT:    mov.b64 {%dummy,%r2}, %rd1;
-; SM35-NEXT:    }
-; SM35-NEXT:    ld.param.u32 %r3, [rotateright64_param_1];
-; SM35-NEXT:    shf.r.wrap.b32 %r4, %r2, %r1, %r3;
-; SM35-NEXT:    shf.r.wrap.b32 %r5, %r1, %r2, %r3;
-; SM35-NEXT:    mov.b64 %rd2, {%r5, %r4};
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    ld.param.u32 %r1, [rotateright64_param_1];
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    shr.u64 %rd2, %rd1, %r2;
+; SM35-NEXT:    neg.s32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shl.b64 %rd3, %rd1, %r4;
+; SM35-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.nvvm.rotate.right.b64(i64 %a, i32 %b)
   ret i64 %val
@@ -148,18 +133,14 @@ define i64 @rotateright64(i64 %a, i32 %b) {
 define i32 @rotl0(i32 %x) {
 ; SM20-LABEL: rotl0(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<3>;
+; SM20-NEXT:    .reg .b32 %r<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u32 %r1, [rotl0_param_0];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b32 %lhs;
-; SM20-NEXT:    .reg .b32 %rhs;
-; SM20-NEXT:    shl.b32 %lhs, %r1, 8;
-; SM20-NEXT:    shr.b32 %rhs, %r1, 24;
-; SM20-NEXT:    add.u32 %r2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b32 [func_retval0+0], %r2;
+; SM20-NEXT:    shr.u32 %r2, %r1, 24;
+; SM20-NEXT:    shl.b32 %r3, %r1, 8;
+; SM20-NEXT:    or.b32 %r4, %r3, %r2;
+; SM20-NEXT:    st.param.b32 [func_retval0+0], %r4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotl0(
@@ -177,51 +158,40 @@ define i32 @rotl0(i32 %x) {
   ret i32 %t2
 }
 
-declare i64 @llvm.fshl.i64(i64, i64, i64)
-declare i64 @llvm.fshr.i64(i64, i64, i64)
-
 ; SM35: rotl64
 define i64 @rotl64(i64 %a, i64 %n) {
 ; SM20-LABEL: rotl64(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<2>;
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotl64_param_0];
 ; SM20-NEXT:    ld.param.u32 %r1, [rotl64_param_1];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    .reg .u32 %amt2;
-; SM20-NEXT:    and.b32 %amt2, %r1, 63;
-; SM20-NEXT:    shl.b64 %lhs, %rd1, %amt2;
-; SM20-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM20-NEXT:    shr.b64 %rhs, %rd1, %amt2;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM20-NEXT:    neg.s32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shr.u64 %rd3, %rd1, %r4;
+; SM20-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotl64(
 ; SM35:       {
-; SM35-NEXT:    .reg .b32 %r<2>;
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotl64_param_0];
 ; SM35-NEXT:    ld.param.u32 %r1, [rotl64_param_1];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b64 %lhs;
-; SM35-NEXT:    .reg .b64 %rhs;
-; SM35-NEXT:    .reg .u32 %amt2;
-; SM35-NEXT:    and.b32 %amt2, %r1, 63;
-; SM35-NEXT:    shl.b64 %lhs, %rd1, %amt2;
-; SM35-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM35-NEXT:    shr.b64 %rhs, %rd1, %amt2;
-; SM35-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM35-NEXT:    }
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM35-NEXT:    neg.s32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shr.u64 %rd3, %rd1, %r4;
+; SM35-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 %n)
   ret i64 %val
@@ -231,34 +201,26 @@ define i64 @rotl64(i64 %a, i64 %n) {
 define i64 @rotl64_imm(i64 %a) {
 ; SM20-LABEL: rotl64_imm(
 ; SM20:       {
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotl64_imm_param_0];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    shl.b64 %lhs, %rd1, 2;
-; SM20-NEXT:    shr.b64 %rhs, %rd1, 62;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    shr.u64 %rd2, %rd1, 62;
+; SM20-NEXT:    shl.b64 %rd3, %rd1, 2;
+; SM20-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotl64_imm(
 ; SM35:       {
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotl64_imm_param_0];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b64 %lhs;
-; SM35-NEXT:    .reg .b64 %rhs;
-; SM35-NEXT:    shl.b64 %lhs, %rd1, 2;
-; SM35-NEXT:    shr.b64 %rhs, %rd1, 62;
-; SM35-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM35-NEXT:    }
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    shr.u64 %rd2, %rd1, 62;
+; SM35-NEXT:    shl.b64 %rd3, %rd1, 2;
+; SM35-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 66)
   ret i64 %val
@@ -268,44 +230,36 @@ define i64 @rotl64_imm(i64 %a) {
 define i64 @rotr64(i64 %a, i64 %n) {
 ; SM20-LABEL: rotr64(
 ; SM20:       {
-; SM20-NEXT:    .reg .b32 %r<2>;
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotr64_param_0];
 ; SM20-NEXT:    ld.param.u32 %r1, [rotr64_param_1];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    .reg .u32 %amt2;
-; SM20-NEXT:    and.b32 %amt2, %r1, 63;
-; SM20-NEXT:    shr.b64 %lhs, %rd1, %amt2;
-; SM20-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM20-NEXT:    shl.b64 %rhs, %rd1, %amt2;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    shr.u64 %rd2, %rd1, %r2;
+; SM20-NEXT:    neg.s32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shl.b64 %rd3, %rd1, %r4;
+; SM20-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotr64(
 ; SM35:       {
-; SM35-NEXT:    .reg .b32 %r<2>;
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotr64_param_0];
 ; SM35-NEXT:    ld.param.u32 %r1, [rotr64_param_1];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b64 %lhs;
-; SM35-NEXT:    .reg .b64 %rhs;
-; SM35-NEXT:    .reg .u32 %amt2;
-; SM35-NEXT:    and.b32 %amt2, %r1, 63;
-; SM35-NEXT:    shr.b64 %lhs, %rd1, %amt2;
-; SM35-NEXT:    sub.u32 %amt2, 64, %amt2;
-; SM35-NEXT:    shl.b64 %rhs, %rd1, %amt2;
-; SM35-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM35-NEXT:    }
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    shr.u64 %rd2, %rd1, %r2;
+; SM35-NEXT:    neg.s32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shl.b64 %rd3, %rd1, %r4;
+; SM35-NEXT:    or.b64 %rd4, %rd2, %rd3;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 %n)
   ret i64 %val
@@ -315,35 +269,180 @@ define i64 @rotr64(i64 %a, i64 %n) {
 define i64 @rotr64_imm(i64 %a) {
 ; SM20-LABEL: rotr64_imm(
 ; SM20:       {
-; SM20-NEXT:    .reg .b64 %rd<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.u64 %rd1, [rotr64_imm_param_0];
-; SM20-NEXT:    {
-; SM20-NEXT:    .reg .b64 %lhs;
-; SM20-NEXT:    .reg .b64 %rhs;
-; SM20-NEXT:    shl.b64 %lhs, %rd1, 62;
-; SM20-NEXT:    shr.b64 %rhs, %rd1, 2;
-; SM20-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM20-NEXT:    }
-; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM20-NEXT:    shl.b64 %rd2, %rd1, 62;
+; SM20-NEXT:    shr.u64 %rd3, %rd1, 2;
+; SM20-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: rotr64_imm(
 ; SM35:       {
-; SM35-NEXT:    .reg .b64 %rd<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.u64 %rd1, [rotr64_imm_param_0];
-; SM35-NEXT:    {
-; SM35-NEXT:    .reg .b64 %lhs;
-; SM35-NEXT:    .reg .b64 %rhs;
-; SM35-NEXT:    shl.b64 %lhs, %rd1, 62;
-; SM35-NEXT:    shr.b64 %rhs, %rd1, 2;
-; SM35-NEXT:    add.u64 %rd2, %lhs, %rhs;
-; SM35-NEXT:    }
-; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd2;
+; SM35-NEXT:    shl.b64 %rd2, %rd1, 62;
+; SM35-NEXT:    shr.u64 %rd3, %rd1, 2;
+; SM35-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd4;
 ; SM35-NEXT:    ret;
   %val = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 66)
   ret i64 %val
 }
+
+define i32 @funnel_shift_right_32(i32 %a, i32 %b, i32 %c) {
+; SM20-LABEL: funnel_shift_right_32(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<11>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.u32 %r1, [funnel_shift_right_32_param_0];
+; SM20-NEXT:    ld.param.u32 %r2, [funnel_shift_right_32_param_2];
+; SM20-NEXT:    and.b32 %r3, %r2, 31;
+; SM20-NEXT:    ld.param.u32 %r4, [funnel_shift_right_32_param_1];
+; SM20-NEXT:    shr.u32 %r5, %r4, %r3;
+; SM20-NEXT:    shl.b32 %r6, %r1, 1;
+; SM20-NEXT:    not.b32 %r7, %r2;
+; SM20-NEXT:    and.b32 %r8, %r7, 31;
+; SM20-NEXT:    shl.b32 %r9, %r6, %r8;
+; SM20-NEXT:    or.b32 %r10, %r9, %r5;
+; SM20-NEXT:    st.param.b32 [func_retval0+0], %r10;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: funnel_shift_right_32(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.u32 %r1, [funnel_shift_right_32_param_0];
+; SM35-NEXT:    ld.param.u32 %r2, [funnel_shift_right_32_param_1];
+; SM35-NEXT:    ld.param.u32 %r3, [funnel_shift_right_32_param_2];
+; SM35-NEXT:    shf.r.wrap.b32 %r4, %r2, %r1, %r3;
+; SM35-NEXT:    st.param.b32 [func_retval0+0], %r4;
+; SM35-NEXT:    ret;
+  %val = call i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %val
+}
+
+define i32 @funnel_shift_left_32(i32 %a, i32 %b, i32 %c) {
+; SM20-LABEL: funnel_shift_left_32(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<11>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.u32 %r1, [funnel_shift_left_32_param_0];
+; SM20-NEXT:    ld.param.u32 %r2, [funnel_shift_left_32_param_2];
+; SM20-NEXT:    and.b32 %r3, %r2, 31;
+; SM20-NEXT:    shl.b32 %r4, %r1, %r3;
+; SM20-NEXT:    ld.param.u32 %r5, [funnel_shift_left_32_param_1];
+; SM20-NEXT:    shr.u32 %r6, %r5, 1;
+; SM20-NEXT:    not.b32 %r7, %r2;
+; SM20-NEXT:    and.b32 %r8, %r7, 31;
+; SM20-NEXT:    shr.u32 %r9, %r6, %r8;
+; SM20-NEXT:    or.b32 %r10, %r4, %r9;
+; SM20-NEXT:    st.param.b32 [func_retval0+0], %r10;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: funnel_shift_left_32(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.u32 %r1, [funnel_shift_left_32_param_0];
+; SM35-NEXT:    ld.param.u32 %r2, [funnel_shift_left_32_param_1];
+; SM35-NEXT:    ld.param.u32 %r3, [funnel_shift_left_32_param_2];
+; SM35-NEXT:    shf.l.wrap.b32 %r4, %r2, %r1, %r3;
+; SM35-NEXT:    st.param.b32 [func_retval0+0], %r4;
+; SM35-NEXT:    ret;
+  %val = call i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+  ret i32 %val
+}
+
+define i64 @funnel_shift_right_64(i64 %a, i64 %b, i64 %c) {
+; SM20-LABEL: funnel_shift_right_64(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.u64 %rd1, [funnel_shift_right_64_param_0];
+; SM20-NEXT:    ld.param.u32 %r1, [funnel_shift_right_64_param_2];
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    ld.param.u64 %rd2, [funnel_shift_right_64_param_1];
+; SM20-NEXT:    shr.u64 %rd3, %rd2, %r2;
+; SM20-NEXT:    shl.b64 %rd4, %rd1, 1;
+; SM20-NEXT:    not.b32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shl.b64 %rd5, %rd4, %r4;
+; SM20-NEXT:    or.b64 %rd6, %rd5, %rd3;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd6;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: funnel_shift_right_64(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.u64 %rd1, [funnel_shift_right_64_param_0];
+; SM35-NEXT:    ld.param.u32 %r1, [funnel_shift_right_64_param_2];
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    ld.param.u64 %rd2, [funnel_shift_right_64_param_1];
+; SM35-NEXT:    shr.u64 %rd3, %rd2, %r2;
+; SM35-NEXT:    shl.b64 %rd4, %rd1, 1;
+; SM35-NEXT:    not.b32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shl.b64 %rd5, %rd4, %r4;
+; SM35-NEXT:    or.b64 %rd6, %rd5, %rd3;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd6;
+; SM35-NEXT:    ret;
+  %val = call i64 @llvm.fshr.i64(i64 %a, i64 %b, i64 %c)
+  ret i64 %val
+}
+
+define i64 @funnel_shift_left_64(i64 %a, i64 %b, i64 %c) {
+; SM20-LABEL: funnel_shift_left_64(
+; SM20:       {
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-EMPTY:
+; SM20-NEXT:  // %bb.0:
+; SM20-NEXT:    ld.param.u64 %rd1, [funnel_shift_left_64_param_0];
+; SM20-NEXT:    ld.param.u32 %r1, [funnel_shift_left_64_param_2];
+; SM20-NEXT:    and.b32 %r2, %r1, 63;
+; SM20-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM20-NEXT:    ld.param.u64 %rd3, [funnel_shift_left_64_param_1];
+; SM20-NEXT:    shr.u64 %rd4, %rd3, 1;
+; SM20-NEXT:    not.b32 %r3, %r1;
+; SM20-NEXT:    and.b32 %r4, %r3, 63;
+; SM20-NEXT:    shr.u64 %rd5, %rd4, %r4;
+; SM20-NEXT:    or.b64 %rd6, %rd2, %rd5;
+; SM20-NEXT:    st.param.b64 [func_retval0+0], %rd6;
+; SM20-NEXT:    ret;
+;
+; SM35-LABEL: funnel_shift_left_64(
+; SM35:       {
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-EMPTY:
+; SM35-NEXT:  // %bb.0:
+; SM35-NEXT:    ld.param.u64 %rd1, [funnel_shift_left_64_param_0];
+; SM35-NEXT:    ld.param.u32 %r1, [funnel_shift_left_64_param_2];
+; SM35-NEXT:    and.b32 %r2, %r1, 63;
+; SM35-NEXT:    shl.b64 %rd2, %rd1, %r2;
+; SM35-NEXT:    ld.param.u64 %rd3, [funnel_shift_left_64_param_1];
+; SM35-NEXT:    shr.u64 %rd4, %rd3, 1;
+; SM35-NEXT:    not.b32 %r3, %r1;
+; SM35-NEXT:    and.b32 %r4, %r3, 63;
+; SM35-NEXT:    shr.u64 %rd5, %rd4, %r4;
+; SM35-NEXT:    or.b64 %rd6, %rd2, %rd5;
+; SM35-NEXT:    st.param.b64 [func_retval0+0], %rd6;
+; SM35-NEXT:    ret;
+  %val = call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 %c)
+  ret i64 %val
+}
+
diff --git a/llvm/test/CodeGen/NVPTX/rotate_64.ll b/llvm/test/CodeGen/NVPTX/rotate_64.ll
index 64659ce1b5c56..05fdb02ac7479 100644
--- a/llvm/test/CodeGen/NVPTX/rotate_64.ll
+++ b/llvm/test/CodeGen/NVPTX/rotate_64.ll
@@ -1,25 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc < %s -march=nvptx64 | FileCheck %s
 ; RUN: %if ptxas %{ llc < %s -march=nvptx64 | %ptxas-verify %}
 
 declare i64 @llvm.nvvm.rotate.b64(i64, i32)
 declare i64 @llvm.nvvm.rotate.right.b64(i64, i32)
 
-; CHECK: rotate64
 define i64 @rotate64(i64 %a, i32 %b) {
-; CHECK: shl.b64         [[LHS:%.*]], [[RD1:%.*]], 3;
-; CHECK: shr.b64         [[RHS:%.*]], [[RD1]], 61;
-; CHECK: add.u64         [[RD2:%.*]], [[LHS]], [[RHS]];
-; CHECK: ret
+; CHECK-LABEL: rotate64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [rotate64_param_0];
+; CHECK-NEXT:    shr.u64 %rd2, %rd1, 61;
+; CHECK-NEXT:    shl.b64 %rd3, %rd1, 3;
+; CHECK-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; CHECK-NEXT:    st.param.b64 [func_retval0+0], %rd4;
+; CHECK-NEXT:    ret;
   %val = tail call i64 @llvm.nvvm.rotate.b64(i64 %a, i32 3)
   ret i64 %val
 }
 
-; CHECK: rotateright64
 define i64 @rotateright64(i64 %a, i32 %b) {
-; CHECK: shl.b64         [[LHS:%.*]], [[RD1:%.*]], 61;
-; CHECK: shr.b64         [[RHS:%.*]], [[RD1]], 3;
-; CHECK: add.u64         [[RD2:%.*]], [[LHS]], [[RHS]];
-; CHECK: ret
+; CHECK-LABEL: rotateright64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.u64 %rd1, [rotateright64_param_0];
+; CHECK-NEXT:    shl.b64 %rd2, %rd1, 61;
+; CHECK-NEXT:    shr.u64 %rd3, %rd1, 3;
+; CHECK-NEXT:    or.b64 %rd4, %rd3, %rd2;
+; CHECK-NEXT:    st.param.b64 [func_retval0+0], %rd4;
+; CHECK-NEXT:    ret;
   %val = tail call i64 @llvm.nvvm.rotate.right.b64(i64 %a, i32 3)
   ret i64 %val
 }
diff --git a/llvm/test/CodeGen/RISCV/calls.ll b/llvm/test/CodeGen/RISCV/calls.ll
index 598a026fb9552..f18bbb4ed84ee 100644
--- a/llvm/test/CodeGen/RISCV/calls.ll
+++ b/llvm/test/CodeGen/RISCV/calls.ll
@@ -11,6 +11,8 @@
 ; RUN:   | FileCheck -check-prefix=RV64I-MEDIUM %s
 ; RUN: llc -code-model=large -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I-LARGE %s
+; RUN: llc -code-model=large -mtriple=riscv64 -mattr=experimental-zicfilp -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I-LARGE-ZICFILP %s
 
 declare i32 @external_function(i32)
 
@@ -62,6 +64,19 @@ define i32 @test_call_external(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_external:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi0:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI0_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi0)(a1)
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 @external_function(i32 %a)
   ret i32 %1
 }
@@ -116,6 +131,19 @@ define i32 @test_call_dso_local(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_dso_local:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi1:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI1_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi1)(a1)
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 @dso_local_function(i32 %a)
   ret i32 %1
 }
@@ -145,6 +173,12 @@ define i32 @defined_function(i32 %a) nounwind {
 ; RV64I-LARGE:       # %bb.0:
 ; RV64I-LARGE-NEXT:    addiw a0, a0, 1
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: defined_function:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addiw a0, a0, 1
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = add i32 %a, 1
   ret i32 %1
 }
@@ -197,6 +231,19 @@ define i32 @test_call_defined(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_defined:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi2:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI3_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi2)(a1)
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 @defined_function(i32 %a)
   ret i32 %1
 }
@@ -256,6 +303,18 @@ define i32 @test_call_indirect(ptr %a, i32 %b) nounwind {
 ; RV64I-LARGE-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_indirect:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    mv a2, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, a1
+; RV64I-LARGE-ZICFILP-NEXT:    jalr a2
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 %a(i32 %b)
   ret i32 %1
 }
@@ -347,6 +406,24 @@ define i32 @test_call_indirect_no_t0(ptr %a, i32 %b, i32 %c, i32 %d, i32 %e, i32
 ; RV64I-LARGE-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_indirect_no_t0:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    mv t1, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, a1
+; RV64I-LARGE-ZICFILP-NEXT:    mv a1, a2
+; RV64I-LARGE-ZICFILP-NEXT:    mv a2, a3
+; RV64I-LARGE-ZICFILP-NEXT:    mv a3, a4
+; RV64I-LARGE-ZICFILP-NEXT:    mv a4, a5
+; RV64I-LARGE-ZICFILP-NEXT:    mv a5, a6
+; RV64I-LARGE-ZICFILP-NEXT:    mv a6, a7
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t1
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 %a(i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h)
   ret i32 %1
 }
@@ -379,6 +456,12 @@ define fastcc i32 @fastcc_function(i32 %a, i32 %b) nounwind {
 ; RV64I-LARGE:       # %bb.0:
 ; RV64I-LARGE-NEXT:    addw a0, a0, a1
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: fastcc_function:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addw a0, a0, a1
+; RV64I-LARGE-ZICFILP-NEXT:    ret
  %1 = add i32 %a, %b
  ret i32 %1
 }
@@ -452,6 +535,24 @@ define i32 @test_call_fastcc(i32 %a, i32 %b) nounwind {
 ; RV64I-LARGE-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 16
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_fastcc:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    mv s0, a0
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi3:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI7_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, s0
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, s0
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call fastcc i32 @fastcc_function(i32 %a, i32 %b)
   ret i32 %a
 }
@@ -572,6 +673,33 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 32
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_external_many_args:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -32
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:    mv s0, a0
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi4:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI8_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi4)(a0)
+; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 8(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    sd s0, 0(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a1, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a2, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a3, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a4, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a5, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a6, s0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a7, s0
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    mv a0, s0
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 @external_many_args(i32 %a, i32 %a, i32 %a, i32 %a, i32 %a,
                                     i32 %a, i32 %a, i32 %a, i32 %a, i32 %a)
   ret i32 %a
@@ -607,6 +735,13 @@ define i32 @defined_many_args(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 %
 ; RV64I-LARGE-NEXT:    lw a0, 8(sp)
 ; RV64I-LARGE-NEXT:    addiw a0, a0, 1
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: defined_many_args:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    lw a0, 8(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    addiw a0, a0, 1
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %added = add i32 %j, 1
   ret i32 %added
 }
@@ -704,6 +839,28 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV64I-LARGE-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-LARGE-NEXT:    addi sp, sp, 32
 ; RV64I-LARGE-NEXT:    ret
+;
+; RV64I-LARGE-ZICFILP-LABEL: test_call_defined_many_args:
+; RV64I-LARGE-ZICFILP:       # %bb.0:
+; RV64I-LARGE-ZICFILP-NEXT:    lpad 0
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, -32
+; RV64I-LARGE-ZICFILP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-LARGE-ZICFILP-NEXT:  .Lpcrel_hi5:
+; RV64I-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI10_0)
+; RV64I-LARGE-ZICFILP-NEXT:    ld t2, %pcrel_lo(.Lpcrel_hi5)(a1)
+; RV64I-LARGE-ZICFILP-NEXT:    sd a0, 8(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    sd a0, 0(sp)
+; RV64I-LARGE-ZICFILP-NEXT:    mv a1, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a2, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a3, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a4, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a5, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a6, a0
+; RV64I-LARGE-ZICFILP-NEXT:    mv a7, a0
+; RV64I-LARGE-ZICFILP-NEXT:    jalr t2
+; RV64I-LARGE-ZICFILP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
+; RV64I-LARGE-ZICFILP-NEXT:    ret
   %1 = call i32 @defined_many_args(i32 %a, i32 %a, i32 %a, i32 %a, i32 %a,
                                    i32 %a, i32 %a, i32 %a, i32 %a, i32 %a)
   ret i32 %1
diff --git a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index ad81db75f7bc9..4831f0b24c7fe 100644
--- a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -1,14 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi=ilp32f -code-model=small -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV32I-SMALL
-; RUN: llc -mtriple=riscv32 -mattr=+f -target-abi=ilp32f -code-model=medium -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV32I-MEDIUM
-; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi=lp64f -code-model=small -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV64I-SMALL
-; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi=lp64f -code-model=medium -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV64I-MEDIUM
-; RUN: llc -mtriple=riscv64 -mattr=+f -target-abi=lp64f -code-model=large -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV64I-LARGE
+; RUN: llc -mtriple=riscv32 -mattr=+f,+zfh -target-abi=ilp32f -code-model=small -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-SMALL,RV32F-SMALL
+; RUN: llc -mtriple=riscv32 -mattr=+f,+zfh -target-abi=ilp32f -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-MEDIUM,RV32F-MEDIUM
+; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh -target-abi=lp64f -code-model=small -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-SMALL,RV64F-SMALL
+; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh -target-abi=lp64f -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-MEDIUM,RV64F-MEDIUM
+; RUN: llc -mtriple=riscv64 -mattr=+f,+zfh -target-abi=lp64f -code-model=large -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-LARGE,RV64F-LARGE
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx,+zhinx -target-abi=ilp32 -code-model=small -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-SMALL,RV32FINX-SMALL
+; RUN: llc -mtriple=riscv32 -mattr=+zfinx,+zhinx -target-abi=ilp32 -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I-MEDIUM,RV32FINX-MEDIUM
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx,+zhinx -target-abi=lp64 -code-model=small -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-SMALL,RV64FINX-SMALL
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx,+zhinx -target-abi=lp64 -code-model=medium -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-MEDIUM,RV64FINX-MEDIUM
+; RUN: llc -mtriple=riscv64 -mattr=+zfinx,+zhinx -target-abi=lp64 -code-model=large -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV64I-LARGE,RV64FINX-LARGE
 
 ; Check lowering of globals
 @G = global i32 0
@@ -238,43 +248,78 @@ indirectgoto:
 ; Check lowering of constantpools
 
 define float @lower_constantpool(float %a) nounwind {
-; RV32I-SMALL-LABEL: lower_constantpool:
-; RV32I-SMALL:       # %bb.0:
-; RV32I-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
-; RV32I-SMALL-NEXT:    flw fa5, %lo(.LCPI3_0)(a0)
-; RV32I-SMALL-NEXT:    fadd.s fa0, fa0, fa5
-; RV32I-SMALL-NEXT:    ret
+; RV32F-SMALL-LABEL: lower_constantpool:
+; RV32F-SMALL:       # %bb.0:
+; RV32F-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
+; RV32F-SMALL-NEXT:    flw fa5, %lo(.LCPI3_0)(a0)
+; RV32F-SMALL-NEXT:    fadd.s fa0, fa0, fa5
+; RV32F-SMALL-NEXT:    ret
 ;
-; RV32I-MEDIUM-LABEL: lower_constantpool:
-; RV32I-MEDIUM:       # %bb.0:
-; RV32I-MEDIUM-NEXT:  .Lpcrel_hi3:
-; RV32I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
-; RV32I-MEDIUM-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
-; RV32I-MEDIUM-NEXT:    fadd.s fa0, fa0, fa5
-; RV32I-MEDIUM-NEXT:    ret
+; RV32F-MEDIUM-LABEL: lower_constantpool:
+; RV32F-MEDIUM:       # %bb.0:
+; RV32F-MEDIUM-NEXT:  .Lpcrel_hi3:
+; RV32F-MEDIUM-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
+; RV32F-MEDIUM-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV32F-MEDIUM-NEXT:    fadd.s fa0, fa0, fa5
+; RV32F-MEDIUM-NEXT:    ret
 ;
-; RV64I-SMALL-LABEL: lower_constantpool:
-; RV64I-SMALL:       # %bb.0:
-; RV64I-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
-; RV64I-SMALL-NEXT:    flw fa5, %lo(.LCPI3_0)(a0)
-; RV64I-SMALL-NEXT:    fadd.s fa0, fa0, fa5
-; RV64I-SMALL-NEXT:    ret
+; RV64F-SMALL-LABEL: lower_constantpool:
+; RV64F-SMALL:       # %bb.0:
+; RV64F-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
+; RV64F-SMALL-NEXT:    flw fa5, %lo(.LCPI3_0)(a0)
+; RV64F-SMALL-NEXT:    fadd.s fa0, fa0, fa5
+; RV64F-SMALL-NEXT:    ret
 ;
-; RV64I-MEDIUM-LABEL: lower_constantpool:
-; RV64I-MEDIUM:       # %bb.0:
-; RV64I-MEDIUM-NEXT:  .Lpcrel_hi3:
-; RV64I-MEDIUM-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
-; RV64I-MEDIUM-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
-; RV64I-MEDIUM-NEXT:    fadd.s fa0, fa0, fa5
-; RV64I-MEDIUM-NEXT:    ret
+; RV64F-MEDIUM-LABEL: lower_constantpool:
+; RV64F-MEDIUM:       # %bb.0:
+; RV64F-MEDIUM-NEXT:  .Lpcrel_hi3:
+; RV64F-MEDIUM-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
+; RV64F-MEDIUM-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV64F-MEDIUM-NEXT:    fadd.s fa0, fa0, fa5
+; RV64F-MEDIUM-NEXT:    ret
 ;
-; RV64I-LARGE-LABEL: lower_constantpool:
-; RV64I-LARGE:       # %bb.0:
-; RV64I-LARGE-NEXT:  .Lpcrel_hi3:
-; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
-; RV64I-LARGE-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
-; RV64I-LARGE-NEXT:    fadd.s fa0, fa0, fa5
-; RV64I-LARGE-NEXT:    ret
+; RV64F-LARGE-LABEL: lower_constantpool:
+; RV64F-LARGE:       # %bb.0:
+; RV64F-LARGE-NEXT:  .Lpcrel_hi3:
+; RV64F-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
+; RV64F-LARGE-NEXT:    flw fa5, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV64F-LARGE-NEXT:    fadd.s fa0, fa0, fa5
+; RV64F-LARGE-NEXT:    ret
+;
+; RV32FINX-SMALL-LABEL: lower_constantpool:
+; RV32FINX-SMALL:       # %bb.0:
+; RV32FINX-SMALL-NEXT:    lui a1, 260097
+; RV32FINX-SMALL-NEXT:    addi a1, a1, -2048
+; RV32FINX-SMALL-NEXT:    fadd.s a0, a0, a1
+; RV32FINX-SMALL-NEXT:    ret
+;
+; RV32FINX-MEDIUM-LABEL: lower_constantpool:
+; RV32FINX-MEDIUM:       # %bb.0:
+; RV32FINX-MEDIUM-NEXT:    lui a1, 260097
+; RV32FINX-MEDIUM-NEXT:    addi a1, a1, -2048
+; RV32FINX-MEDIUM-NEXT:    fadd.s a0, a0, a1
+; RV32FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-SMALL-LABEL: lower_constantpool:
+; RV64FINX-SMALL:       # %bb.0:
+; RV64FINX-SMALL-NEXT:    lui a1, 260097
+; RV64FINX-SMALL-NEXT:    addiw a1, a1, -2048
+; RV64FINX-SMALL-NEXT:    fadd.s a0, a0, a1
+; RV64FINX-SMALL-NEXT:    ret
+;
+; RV64FINX-MEDIUM-LABEL: lower_constantpool:
+; RV64FINX-MEDIUM:       # %bb.0:
+; RV64FINX-MEDIUM-NEXT:    lui a1, 260097
+; RV64FINX-MEDIUM-NEXT:    addiw a1, a1, -2048
+; RV64FINX-MEDIUM-NEXT:    fadd.s a0, a0, a1
+; RV64FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-LARGE-LABEL: lower_constantpool:
+; RV64FINX-LARGE:       # %bb.0:
+; RV64FINX-LARGE-NEXT:    lui a1, 260097
+; RV64FINX-LARGE-NEXT:    addiw a1, a1, -2048
+; RV64FINX-LARGE-NEXT:    fadd.s a0, a0, a1
+; RV64FINX-LARGE-NEXT:    ret
   %1 = fadd float %a, 1.000244140625
   ret float %1
 }
@@ -289,13 +334,13 @@ define i32 @lower_extern_weak(i32 %a) nounwind {
 ; RV32I-SMALL-NEXT:    lw a0, %lo(W)(a0)
 ; RV32I-SMALL-NEXT:    ret
 ;
-; RV32I-MEDIUM-LABEL: lower_extern_weak:
-; RV32I-MEDIUM:       # %bb.0:
-; RV32I-MEDIUM-NEXT:  .Lpcrel_hi4:
-; RV32I-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
-; RV32I-MEDIUM-NEXT:    lw a0, %pcrel_lo(.Lpcrel_hi4)(a0)
-; RV32I-MEDIUM-NEXT:    lw a0, 0(a0)
-; RV32I-MEDIUM-NEXT:    ret
+; RV32F-MEDIUM-LABEL: lower_extern_weak:
+; RV32F-MEDIUM:       # %bb.0:
+; RV32F-MEDIUM-NEXT:  .Lpcrel_hi4:
+; RV32F-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
+; RV32F-MEDIUM-NEXT:    lw a0, %pcrel_lo(.Lpcrel_hi4)(a0)
+; RV32F-MEDIUM-NEXT:    lw a0, 0(a0)
+; RV32F-MEDIUM-NEXT:    ret
 ;
 ; RV64I-SMALL-LABEL: lower_extern_weak:
 ; RV64I-SMALL:       # %bb.0:
@@ -303,21 +348,130 @@ define i32 @lower_extern_weak(i32 %a) nounwind {
 ; RV64I-SMALL-NEXT:    lw a0, %lo(W)(a0)
 ; RV64I-SMALL-NEXT:    ret
 ;
-; RV64I-MEDIUM-LABEL: lower_extern_weak:
-; RV64I-MEDIUM:       # %bb.0:
-; RV64I-MEDIUM-NEXT:  .Lpcrel_hi4:
-; RV64I-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
-; RV64I-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi4)(a0)
-; RV64I-MEDIUM-NEXT:    lw a0, 0(a0)
-; RV64I-MEDIUM-NEXT:    ret
+; RV64F-MEDIUM-LABEL: lower_extern_weak:
+; RV64F-MEDIUM:       # %bb.0:
+; RV64F-MEDIUM-NEXT:  .Lpcrel_hi4:
+; RV64F-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
+; RV64F-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi4)(a0)
+; RV64F-MEDIUM-NEXT:    lw a0, 0(a0)
+; RV64F-MEDIUM-NEXT:    ret
 ;
-; RV64I-LARGE-LABEL: lower_extern_weak:
-; RV64I-LARGE:       # %bb.0:
-; RV64I-LARGE-NEXT:  .Lpcrel_hi4:
-; RV64I-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI4_0)
-; RV64I-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi4)(a0)
-; RV64I-LARGE-NEXT:    lw a0, 0(a0)
-; RV64I-LARGE-NEXT:    ret
+; RV64F-LARGE-LABEL: lower_extern_weak:
+; RV64F-LARGE:       # %bb.0:
+; RV64F-LARGE-NEXT:  .Lpcrel_hi4:
+; RV64F-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI4_0)
+; RV64F-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi4)(a0)
+; RV64F-LARGE-NEXT:    lw a0, 0(a0)
+; RV64F-LARGE-NEXT:    ret
+;
+; RV32FINX-MEDIUM-LABEL: lower_extern_weak:
+; RV32FINX-MEDIUM:       # %bb.0:
+; RV32FINX-MEDIUM-NEXT:  .Lpcrel_hi3:
+; RV32FINX-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
+; RV32FINX-MEDIUM-NEXT:    lw a0, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV32FINX-MEDIUM-NEXT:    lw a0, 0(a0)
+; RV32FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-MEDIUM-LABEL: lower_extern_weak:
+; RV64FINX-MEDIUM:       # %bb.0:
+; RV64FINX-MEDIUM-NEXT:  .Lpcrel_hi3:
+; RV64FINX-MEDIUM-NEXT:    auipc a0, %got_pcrel_hi(W)
+; RV64FINX-MEDIUM-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV64FINX-MEDIUM-NEXT:    lw a0, 0(a0)
+; RV64FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-LARGE-LABEL: lower_extern_weak:
+; RV64FINX-LARGE:       # %bb.0:
+; RV64FINX-LARGE-NEXT:  .Lpcrel_hi3:
+; RV64FINX-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI4_0)
+; RV64FINX-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi3)(a0)
+; RV64FINX-LARGE-NEXT:    lw a0, 0(a0)
+; RV64FINX-LARGE-NEXT:    ret
   %1 = load volatile i32, ptr @W
   ret i32 %1
 }
+
+@X = global half 1.5
+
+define half @lower_global_half(half %a) nounwind {
+; RV32F-SMALL-LABEL: lower_global_half:
+; RV32F-SMALL:       # %bb.0:
+; RV32F-SMALL-NEXT:    lui a0, %hi(X)
+; RV32F-SMALL-NEXT:    flh fa5, %lo(X)(a0)
+; RV32F-SMALL-NEXT:    fadd.h fa0, fa0, fa5
+; RV32F-SMALL-NEXT:    ret
+;
+; RV32F-MEDIUM-LABEL: lower_global_half:
+; RV32F-MEDIUM:       # %bb.0:
+; RV32F-MEDIUM-NEXT:  .Lpcrel_hi5:
+; RV32F-MEDIUM-NEXT:    auipc a0, %pcrel_hi(X)
+; RV32F-MEDIUM-NEXT:    flh fa5, %pcrel_lo(.Lpcrel_hi5)(a0)
+; RV32F-MEDIUM-NEXT:    fadd.h fa0, fa0, fa5
+; RV32F-MEDIUM-NEXT:    ret
+;
+; RV64F-SMALL-LABEL: lower_global_half:
+; RV64F-SMALL:       # %bb.0:
+; RV64F-SMALL-NEXT:    lui a0, %hi(X)
+; RV64F-SMALL-NEXT:    flh fa5, %lo(X)(a0)
+; RV64F-SMALL-NEXT:    fadd.h fa0, fa0, fa5
+; RV64F-SMALL-NEXT:    ret
+;
+; RV64F-MEDIUM-LABEL: lower_global_half:
+; RV64F-MEDIUM:       # %bb.0:
+; RV64F-MEDIUM-NEXT:  .Lpcrel_hi5:
+; RV64F-MEDIUM-NEXT:    auipc a0, %pcrel_hi(X)
+; RV64F-MEDIUM-NEXT:    flh fa5, %pcrel_lo(.Lpcrel_hi5)(a0)
+; RV64F-MEDIUM-NEXT:    fadd.h fa0, fa0, fa5
+; RV64F-MEDIUM-NEXT:    ret
+;
+; RV64F-LARGE-LABEL: lower_global_half:
+; RV64F-LARGE:       # %bb.0:
+; RV64F-LARGE-NEXT:  .Lpcrel_hi5:
+; RV64F-LARGE-NEXT:    auipc a0, %pcrel_hi(.LCPI5_0)
+; RV64F-LARGE-NEXT:    ld a0, %pcrel_lo(.Lpcrel_hi5)(a0)
+; RV64F-LARGE-NEXT:    flh fa5, 0(a0)
+; RV64F-LARGE-NEXT:    fadd.h fa0, fa0, fa5
+; RV64F-LARGE-NEXT:    ret
+;
+; RV32FINX-SMALL-LABEL: lower_global_half:
+; RV32FINX-SMALL:       # %bb.0:
+; RV32FINX-SMALL-NEXT:    lui a1, %hi(X)
+; RV32FINX-SMALL-NEXT:    lh a1, %lo(X)(a1)
+; RV32FINX-SMALL-NEXT:    fadd.h a0, a0, a1
+; RV32FINX-SMALL-NEXT:    ret
+;
+; RV32FINX-MEDIUM-LABEL: lower_global_half:
+; RV32FINX-MEDIUM:       # %bb.0:
+; RV32FINX-MEDIUM-NEXT:  .Lpcrel_hi4:
+; RV32FINX-MEDIUM-NEXT:    auipc a1, %pcrel_hi(X)
+; RV32FINX-MEDIUM-NEXT:    lh a1, %pcrel_lo(.Lpcrel_hi4)(a1)
+; RV32FINX-MEDIUM-NEXT:    fadd.h a0, a0, a1
+; RV32FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-SMALL-LABEL: lower_global_half:
+; RV64FINX-SMALL:       # %bb.0:
+; RV64FINX-SMALL-NEXT:    lui a1, %hi(X)
+; RV64FINX-SMALL-NEXT:    lh a1, %lo(X)(a1)
+; RV64FINX-SMALL-NEXT:    fadd.h a0, a0, a1
+; RV64FINX-SMALL-NEXT:    ret
+;
+; RV64FINX-MEDIUM-LABEL: lower_global_half:
+; RV64FINX-MEDIUM:       # %bb.0:
+; RV64FINX-MEDIUM-NEXT:  .Lpcrel_hi4:
+; RV64FINX-MEDIUM-NEXT:    auipc a1, %pcrel_hi(X)
+; RV64FINX-MEDIUM-NEXT:    lh a1, %pcrel_lo(.Lpcrel_hi4)(a1)
+; RV64FINX-MEDIUM-NEXT:    fadd.h a0, a0, a1
+; RV64FINX-MEDIUM-NEXT:    ret
+;
+; RV64FINX-LARGE-LABEL: lower_global_half:
+; RV64FINX-LARGE:       # %bb.0:
+; RV64FINX-LARGE-NEXT:  .Lpcrel_hi4:
+; RV64FINX-LARGE-NEXT:    auipc a1, %pcrel_hi(.LCPI5_0)
+; RV64FINX-LARGE-NEXT:    ld a1, %pcrel_lo(.Lpcrel_hi4)(a1)
+; RV64FINX-LARGE-NEXT:    lh a1, 0(a1)
+; RV64FINX-LARGE-NEXT:    fadd.h a0, a0, a1
+; RV64FINX-LARGE-NEXT:    ret
+  %b = load half, ptr @X
+  %1 = fadd half %a, %b
+  ret half %1
+}
diff --git a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
index ca40ba0399973..de5bb8a30db16 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-without-f-reg.ll
@@ -246,28 +246,32 @@ define fastcc half @callee_half_32(<32 x half> %A) nounwind {
 define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX32-LABEL: caller_half_32:
 ; ZHINX32:       # %bb.0:
-; ZHINX32-NEXT:    addi sp, sp, -96
-; ZHINX32-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
-; ZHINX32-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    addi sp, sp, -112
+; ZHINX32-NEXT:    sw ra, 108(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s0, 104(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s1, 100(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s2, 96(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s3, 92(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s4, 88(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s5, 84(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s6, 80(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s7, 76(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s8, 72(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s9, 68(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s10, 64(sp) # 4-byte Folded Spill
+; ZHINX32-NEXT:    sw s11, 60(sp) # 4-byte Folded Spill
 ; ZHINX32-NEXT:    lh t0, 112(sp)
-; ZHINX32-NEXT:    lh t1, 116(sp)
-; ZHINX32-NEXT:    lh t2, 120(sp)
-; ZHINX32-NEXT:    lh s0, 124(sp)
-; ZHINX32-NEXT:    lh t3, 128(sp)
-; ZHINX32-NEXT:    lh t4, 132(sp)
-; ZHINX32-NEXT:    lh t5, 136(sp)
-; ZHINX32-NEXT:    lh t6, 140(sp)
+; ZHINX32-NEXT:    sh t0, 58(sp) # 2-byte Folded Spill
+; ZHINX32-NEXT:    lh t0, 116(sp)
+; ZHINX32-NEXT:    sh t0, 56(sp) # 2-byte Folded Spill
+; ZHINX32-NEXT:    lh t0, 120(sp)
+; ZHINX32-NEXT:    sh t0, 54(sp) # 2-byte Folded Spill
+; ZHINX32-NEXT:    lh t0, 124(sp)
+; ZHINX32-NEXT:    sh t0, 52(sp) # 2-byte Folded Spill
+; ZHINX32-NEXT:    lh t6, 128(sp)
+; ZHINX32-NEXT:    lh t5, 132(sp)
+; ZHINX32-NEXT:    lh t4, 136(sp)
+; ZHINX32-NEXT:    lh s0, 140(sp)
 ; ZHINX32-NEXT:    lh s1, 144(sp)
 ; ZHINX32-NEXT:    lh s2, 148(sp)
 ; ZHINX32-NEXT:    lh s3, 152(sp)
@@ -280,122 +284,134 @@ define half @caller_half_32(<32 x half> %A) nounwind {
 ; ZHINX32-NEXT:    lh s10, 180(sp)
 ; ZHINX32-NEXT:    lh s11, 184(sp)
 ; ZHINX32-NEXT:    lh ra, 188(sp)
-; ZHINX32-NEXT:    sh ra, 38(sp)
-; ZHINX32-NEXT:    sh s11, 36(sp)
-; ZHINX32-NEXT:    sh s10, 34(sp)
-; ZHINX32-NEXT:    sh s9, 32(sp)
-; ZHINX32-NEXT:    sh s8, 30(sp)
-; ZHINX32-NEXT:    sh s7, 28(sp)
-; ZHINX32-NEXT:    sh s6, 26(sp)
-; ZHINX32-NEXT:    sh s5, 24(sp)
-; ZHINX32-NEXT:    sh s4, 22(sp)
-; ZHINX32-NEXT:    sh s3, 20(sp)
-; ZHINX32-NEXT:    sh s2, 18(sp)
-; ZHINX32-NEXT:    sh s1, 16(sp)
-; ZHINX32-NEXT:    sh t6, 14(sp)
-; ZHINX32-NEXT:    sh t5, 12(sp)
-; ZHINX32-NEXT:    sh t4, 10(sp)
-; ZHINX32-NEXT:    sh t3, 8(sp)
-; ZHINX32-NEXT:    lh t3, 96(sp)
-; ZHINX32-NEXT:    lh t4, 100(sp)
-; ZHINX32-NEXT:    lh t5, 104(sp)
-; ZHINX32-NEXT:    lh t6, 108(sp)
+; ZHINX32-NEXT:    lh t3, 192(sp)
+; ZHINX32-NEXT:    lh t2, 196(sp)
+; ZHINX32-NEXT:    lh t1, 200(sp)
+; ZHINX32-NEXT:    lh t0, 204(sp)
+; ZHINX32-NEXT:    sh t0, 38(sp)
+; ZHINX32-NEXT:    sh t1, 36(sp)
+; ZHINX32-NEXT:    sh t2, 34(sp)
+; ZHINX32-NEXT:    sh t3, 32(sp)
+; ZHINX32-NEXT:    sh ra, 30(sp)
+; ZHINX32-NEXT:    sh s11, 28(sp)
+; ZHINX32-NEXT:    sh s10, 26(sp)
+; ZHINX32-NEXT:    sh s9, 24(sp)
+; ZHINX32-NEXT:    sh s8, 22(sp)
+; ZHINX32-NEXT:    sh s7, 20(sp)
+; ZHINX32-NEXT:    sh s6, 18(sp)
+; ZHINX32-NEXT:    sh s5, 16(sp)
+; ZHINX32-NEXT:    sh s4, 14(sp)
+; ZHINX32-NEXT:    sh s3, 12(sp)
+; ZHINX32-NEXT:    sh s2, 10(sp)
+; ZHINX32-NEXT:    sh s1, 8(sp)
 ; ZHINX32-NEXT:    sh s0, 6(sp)
-; ZHINX32-NEXT:    sh t2, 4(sp)
-; ZHINX32-NEXT:    sh t1, 2(sp)
-; ZHINX32-NEXT:    sh t0, 0(sp)
+; ZHINX32-NEXT:    sh t4, 4(sp)
+; ZHINX32-NEXT:    sh t5, 2(sp)
+; ZHINX32-NEXT:    sh t6, 0(sp)
+; ZHINX32-NEXT:    lh t3, 58(sp) # 2-byte Folded Reload
+; ZHINX32-NEXT:    lh t4, 56(sp) # 2-byte Folded Reload
+; ZHINX32-NEXT:    lh t5, 54(sp) # 2-byte Folded Reload
+; ZHINX32-NEXT:    lh t6, 52(sp) # 2-byte Folded Reload
 ; ZHINX32-NEXT:    call callee_half_32
-; ZHINX32-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
-; ZHINX32-NEXT:    addi sp, sp, 96
+; ZHINX32-NEXT:    lw ra, 108(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s0, 104(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s1, 100(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s2, 96(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s3, 92(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s4, 88(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s5, 84(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s6, 80(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s7, 76(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s8, 72(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s9, 68(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s10, 64(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    lw s11, 60(sp) # 4-byte Folded Reload
+; ZHINX32-NEXT:    addi sp, sp, 112
 ; ZHINX32-NEXT:    ret
 ;
 ; ZHINX64-LABEL: caller_half_32:
 ; ZHINX64:       # %bb.0:
-; ZHINX64-NEXT:    addi sp, sp, -144
-; ZHINX64-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s0, 128(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s1, 120(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s2, 112(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s3, 104(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s4, 96(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s5, 88(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s6, 80(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s7, 72(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s8, 64(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s9, 56(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s10, 48(sp) # 8-byte Folded Spill
-; ZHINX64-NEXT:    sd s11, 40(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    addi sp, sp, -160
+; ZHINX64-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s2, 128(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s3, 120(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s4, 112(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s5, 104(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s6, 96(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s7, 88(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s8, 80(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
+; ZHINX64-NEXT:    lh t0, 160(sp)
+; ZHINX64-NEXT:    sh t0, 54(sp) # 2-byte Folded Spill
+; ZHINX64-NEXT:    lh t0, 168(sp)
+; ZHINX64-NEXT:    sh t0, 52(sp) # 2-byte Folded Spill
 ; ZHINX64-NEXT:    lh t0, 176(sp)
-; ZHINX64-NEXT:    lh t1, 184(sp)
-; ZHINX64-NEXT:    lh t2, 192(sp)
-; ZHINX64-NEXT:    lh s0, 200(sp)
-; ZHINX64-NEXT:    lh t3, 208(sp)
-; ZHINX64-NEXT:    lh t4, 216(sp)
-; ZHINX64-NEXT:    lh t5, 224(sp)
-; ZHINX64-NEXT:    lh t6, 232(sp)
-; ZHINX64-NEXT:    lh s1, 240(sp)
-; ZHINX64-NEXT:    lh s2, 248(sp)
-; ZHINX64-NEXT:    lh s3, 256(sp)
-; ZHINX64-NEXT:    lh s4, 264(sp)
-; ZHINX64-NEXT:    lh s5, 272(sp)
-; ZHINX64-NEXT:    lh s6, 280(sp)
-; ZHINX64-NEXT:    lh s7, 288(sp)
-; ZHINX64-NEXT:    lh s8, 296(sp)
-; ZHINX64-NEXT:    lh s9, 304(sp)
-; ZHINX64-NEXT:    lh s10, 312(sp)
-; ZHINX64-NEXT:    lh s11, 320(sp)
-; ZHINX64-NEXT:    lh ra, 328(sp)
-; ZHINX64-NEXT:    sh ra, 38(sp)
-; ZHINX64-NEXT:    sh s11, 36(sp)
-; ZHINX64-NEXT:    sh s10, 34(sp)
-; ZHINX64-NEXT:    sh s9, 32(sp)
-; ZHINX64-NEXT:    sh s8, 30(sp)
-; ZHINX64-NEXT:    sh s7, 28(sp)
-; ZHINX64-NEXT:    sh s6, 26(sp)
-; ZHINX64-NEXT:    sh s5, 24(sp)
-; ZHINX64-NEXT:    sh s4, 22(sp)
-; ZHINX64-NEXT:    sh s3, 20(sp)
-; ZHINX64-NEXT:    sh s2, 18(sp)
-; ZHINX64-NEXT:    sh s1, 16(sp)
-; ZHINX64-NEXT:    sh t6, 14(sp)
-; ZHINX64-NEXT:    sh t5, 12(sp)
-; ZHINX64-NEXT:    sh t4, 10(sp)
-; ZHINX64-NEXT:    sh t3, 8(sp)
-; ZHINX64-NEXT:    lh t3, 144(sp)
-; ZHINX64-NEXT:    lh t4, 152(sp)
-; ZHINX64-NEXT:    lh t5, 160(sp)
-; ZHINX64-NEXT:    lh t6, 168(sp)
+; ZHINX64-NEXT:    sh t0, 50(sp) # 2-byte Folded Spill
+; ZHINX64-NEXT:    lh t0, 184(sp)
+; ZHINX64-NEXT:    sh t0, 48(sp) # 2-byte Folded Spill
+; ZHINX64-NEXT:    lh t6, 192(sp)
+; ZHINX64-NEXT:    lh t5, 200(sp)
+; ZHINX64-NEXT:    lh t4, 208(sp)
+; ZHINX64-NEXT:    lh s0, 216(sp)
+; ZHINX64-NEXT:    lh s1, 224(sp)
+; ZHINX64-NEXT:    lh s2, 232(sp)
+; ZHINX64-NEXT:    lh s3, 240(sp)
+; ZHINX64-NEXT:    lh s4, 248(sp)
+; ZHINX64-NEXT:    lh s5, 256(sp)
+; ZHINX64-NEXT:    lh s6, 264(sp)
+; ZHINX64-NEXT:    lh s7, 272(sp)
+; ZHINX64-NEXT:    lh s8, 280(sp)
+; ZHINX64-NEXT:    lh s9, 288(sp)
+; ZHINX64-NEXT:    lh s10, 296(sp)
+; ZHINX64-NEXT:    lh s11, 304(sp)
+; ZHINX64-NEXT:    lh ra, 312(sp)
+; ZHINX64-NEXT:    lh t3, 320(sp)
+; ZHINX64-NEXT:    lh t2, 328(sp)
+; ZHINX64-NEXT:    lh t1, 336(sp)
+; ZHINX64-NEXT:    lh t0, 344(sp)
+; ZHINX64-NEXT:    sh t0, 38(sp)
+; ZHINX64-NEXT:    sh t1, 36(sp)
+; ZHINX64-NEXT:    sh t2, 34(sp)
+; ZHINX64-NEXT:    sh t3, 32(sp)
+; ZHINX64-NEXT:    sh ra, 30(sp)
+; ZHINX64-NEXT:    sh s11, 28(sp)
+; ZHINX64-NEXT:    sh s10, 26(sp)
+; ZHINX64-NEXT:    sh s9, 24(sp)
+; ZHINX64-NEXT:    sh s8, 22(sp)
+; ZHINX64-NEXT:    sh s7, 20(sp)
+; ZHINX64-NEXT:    sh s6, 18(sp)
+; ZHINX64-NEXT:    sh s5, 16(sp)
+; ZHINX64-NEXT:    sh s4, 14(sp)
+; ZHINX64-NEXT:    sh s3, 12(sp)
+; ZHINX64-NEXT:    sh s2, 10(sp)
+; ZHINX64-NEXT:    sh s1, 8(sp)
 ; ZHINX64-NEXT:    sh s0, 6(sp)
-; ZHINX64-NEXT:    sh t2, 4(sp)
-; ZHINX64-NEXT:    sh t1, 2(sp)
-; ZHINX64-NEXT:    sh t0, 0(sp)
+; ZHINX64-NEXT:    sh t4, 4(sp)
+; ZHINX64-NEXT:    sh t5, 2(sp)
+; ZHINX64-NEXT:    sh t6, 0(sp)
+; ZHINX64-NEXT:    lh t3, 54(sp) # 2-byte Folded Reload
+; ZHINX64-NEXT:    lh t4, 52(sp) # 2-byte Folded Reload
+; ZHINX64-NEXT:    lh t5, 50(sp) # 2-byte Folded Reload
+; ZHINX64-NEXT:    lh t6, 48(sp) # 2-byte Folded Reload
 ; ZHINX64-NEXT:    call callee_half_32
-; ZHINX64-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s0, 128(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s1, 120(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s2, 112(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s3, 104(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s4, 96(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s5, 88(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s6, 80(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s7, 72(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s8, 64(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s9, 56(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s10, 48(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    ld s11, 40(sp) # 8-byte Folded Reload
-; ZHINX64-NEXT:    addi sp, sp, 144
+; ZHINX64-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s2, 128(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s3, 120(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s4, 112(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s5, 104(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s6, 96(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s7, 88(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s8, 80(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; ZHINX64-NEXT:    addi sp, sp, 160
 ; ZHINX64-NEXT:    ret
 ;
 ; ZFINX32-LABEL: caller_half_32:
diff --git a/llvm/test/CodeGen/RISCV/half-arith.ll b/llvm/test/CodeGen/RISCV/half-arith.ll
index 27829f2b65759..4c2deafdc7e66 100644
--- a/llvm/test/CodeGen/RISCV/half-arith.ll
+++ b/llvm/test/CodeGen/RISCV/half-arith.ll
@@ -466,20 +466,26 @@ define half @fsgnj_h(half %a, half %b) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: fsgnj_h:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV32IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    or a0, a0, a1
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: fsgnj_h:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV64IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    or a0, a0, a1
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = call half @llvm.copysign.f16(half %a, half %b)
   ret half %1
@@ -725,6 +731,7 @@ define half @fsgnjn_h(half %a, half %b) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: fsgnjn_h:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    fcvt.s.h a1, a1
 ; RV32IZHINXMIN-NEXT:    fcvt.s.h a2, a0
 ; RV32IZHINXMIN-NEXT:    fadd.s a1, a2, a1
@@ -735,10 +742,12 @@ define half @fsgnjn_h(half %a, half %b) nounwind {
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    or a0, a0, a1
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: fsgnjn_h:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    fcvt.s.h a1, a1
 ; RV64IZHINXMIN-NEXT:    fcvt.s.h a2, a0
 ; RV64IZHINXMIN-NEXT:    fadd.s a1, a2, a1
@@ -749,6 +758,7 @@ define half @fsgnjn_h(half %a, half %b) nounwind {
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    or a0, a0, a1
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = fadd half %a, %b
   %2 = fneg half %1
@@ -1702,8 +1712,7 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
 ; CHECKIZHINX-LABEL: fnmadd_h_3:
 ; CHECKIZHINX:       # %bb.0:
 ; CHECKIZHINX-NEXT:    fmadd.h a0, a0, a1, a2
-; CHECKIZHINX-NEXT:    lui a1, 1048568
-; CHECKIZHINX-NEXT:    xor a0, a0, a1
+; CHECKIZHINX-NEXT:    fneg.h a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fnmadd_h_3:
@@ -1798,6 +1807,7 @@ define half @fnmadd_h_3(half %a, half %b, half %c) nounwind {
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    lui a1, 1048568
 ; CHECKIZHINXMIN-NEXT:    xor a0, a0, a1
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
   %neg = fneg half %1
@@ -1823,9 +1833,7 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
 ;
 ; CHECKIZHINX-LABEL: fnmadd_nsz:
 ; CHECKIZHINX:       # %bb.0:
-; CHECKIZHINX-NEXT:    fmadd.h a0, a0, a1, a2
-; CHECKIZHINX-NEXT:    lui a1, 1048568
-; CHECKIZHINX-NEXT:    xor a0, a0, a1
+; CHECKIZHINX-NEXT:    fnmadd.h a0, a0, a1, a2
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fnmadd_nsz:
@@ -1920,6 +1928,7 @@ define half @fnmadd_nsz(half %a, half %b, half %c) nounwind {
 ; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINXMIN-NEXT:    lui a1, 1048568
 ; CHECKIZHINXMIN-NEXT:    xor a0, a0, a1
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECKIZHINXMIN-NEXT:    ret
   %1 = call nsz half @llvm.fma.f16(half %a, half %b, half %c)
   %neg = fneg nsz half %1
@@ -2910,6 +2919,7 @@ define half @fsgnjx_f16(half %x, half %y) nounwind {
 ;
 ; CHECKIZHINXMIN-LABEL: fsgnjx_f16:
 ; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECKIZHINXMIN-NEXT:    lui a2, 1048568
 ; CHECKIZHINXMIN-NEXT:    and a0, a0, a2
 ; CHECKIZHINXMIN-NEXT:    li a2, 15
diff --git a/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll
index 506b7027a8b35..e0c47bfac6fec 100644
--- a/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll
@@ -55,14 +55,12 @@ define half @fneg(half %a) nounwind {
 ;
 ; RV32IZHINX-LABEL: fneg:
 ; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    lui a1, 1048568
-; RV32IZHINX-NEXT:    xor a0, a0, a1
+; RV32IZHINX-NEXT:    fneg.h a0, a0
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: fneg:
 ; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    lui a1, 1048568
-; RV64IZHINX-NEXT:    xor a0, a0, a1
+; RV64IZHINX-NEXT:    fneg.h a0, a0
 ; RV64IZHINX-NEXT:    ret
 ;
 ; RV32IZFHMIN-LABEL: fneg:
@@ -79,8 +77,10 @@ define half @fneg(half %a) nounwind {
 ;
 ; RVIZHINXMIN-LABEL: fneg:
 ; RVIZHINXMIN:       # %bb.0:
+; RVIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RVIZHINXMIN-NEXT:    lui a1, 1048568
 ; RVIZHINXMIN-NEXT:    xor a0, a0, a1
+; RVIZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RVIZHINXMIN-NEXT:    ret
   %1 = fneg half %a
   ret half %1
@@ -115,14 +115,12 @@ define half @fabs(half %a) nounwind {
 ;
 ; RV32IZHINX-LABEL: fabs:
 ; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    slli a0, a0, 17
-; RV32IZHINX-NEXT:    srli a0, a0, 17
+; RV32IZHINX-NEXT:    fabs.h a0, a0
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: fabs:
 ; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    slli a0, a0, 49
-; RV64IZHINX-NEXT:    srli a0, a0, 49
+; RV64IZHINX-NEXT:    fabs.h a0, a0
 ; RV64IZHINX-NEXT:    ret
 ;
 ; RV32IZFHMIN-LABEL: fabs:
@@ -139,14 +137,18 @@ define half @fabs(half %a) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: fabs:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: fabs:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = call half @llvm.fabs.f16(half %a)
   ret half %1
@@ -227,22 +229,28 @@ define half @fcopysign_fneg(half %a, half %b) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: fcopysign_fneg:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    not a1, a1
 ; RV32IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV32IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    or a0, a0, a1
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: fcopysign_fneg:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    not a1, a1
 ; RV64IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV64IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    or a0, a0, a1
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = fneg half %b
   %2 = call half @llvm.copysign.f16(half %a, half %1)
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index e5585661ce79a..0c84a08f1fd45 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -5536,10 +5536,12 @@ define half @bitcast_h_i16(i16 %a) nounwind {
 ;
 ; CHECKIZHINX-LABEL: bitcast_h_i16:
 ; CHECKIZHINX:       # %bb.0:
+; CHECKIZHINX-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; CHECKIZDINXZHINX-LABEL: bitcast_h_i16:
 ; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECKIZDINXZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: bitcast_h_i16:
@@ -5588,18 +5590,22 @@ define half @bitcast_h_i16(i16 %a) nounwind {
 ;
 ; CHECK32-IZHINXMIN-LABEL: bitcast_h_i16:
 ; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECK32-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK64-IZHINXMIN-LABEL: bitcast_h_i16:
 ; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-IZDINXZHINXMIN-LABEL: bitcast_h_i16:
 ; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECK32-IZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK64-IZDINXZHINXMIN-LABEL: bitcast_h_i16:
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
   %1 = bitcast i16 %a to half
   ret half %1
@@ -5623,10 +5629,12 @@ define i16 @bitcast_i16_h(half %a) nounwind {
 ;
 ; CHECKIZHINX-LABEL: bitcast_i16_h:
 ; CHECKIZHINX:       # %bb.0:
+; CHECKIZHINX-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECKIZHINX-NEXT:    ret
 ;
 ; CHECKIZDINXZHINX-LABEL: bitcast_i16_h:
 ; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECKIZDINXZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: bitcast_i16_h:
@@ -5667,18 +5675,22 @@ define i16 @bitcast_i16_h(half %a) nounwind {
 ;
 ; CHECK32-IZHINXMIN-LABEL: bitcast_i16_h:
 ; CHECK32-IZHINXMIN:       # %bb.0:
+; CHECK32-IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECK32-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK64-IZHINXMIN-LABEL: bitcast_i16_h:
 ; CHECK64-IZHINXMIN:       # %bb.0:
+; CHECK64-IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-IZDINXZHINXMIN-LABEL: bitcast_i16_h:
 ; CHECK32-IZDINXZHINXMIN:       # %bb.0:
+; CHECK32-IZDINXZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECK32-IZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK64-IZDINXZHINXMIN-LABEL: bitcast_i16_h:
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0:
+; CHECK64-IZDINXZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
   %1 = bitcast half %a to i16
   ret i16 %1
diff --git a/llvm/test/CodeGen/RISCV/half-imm.ll b/llvm/test/CodeGen/RISCV/half-imm.ll
index 2ebc28c2ebd44..1045df1c3e766 100644
--- a/llvm/test/CodeGen/RISCV/half-imm.ll
+++ b/llvm/test/CodeGen/RISCV/half-imm.ll
@@ -32,12 +32,14 @@ define half @half_imm() nounwind {
 ; RV32IZHINX:       # %bb.0:
 ; RV32IZHINX-NEXT:    lui a0, 4
 ; RV32IZHINX-NEXT:    addi a0, a0, 512
+; RV32IZHINX-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINX-NEXT:    ret
 ;
 ; RV64IZHINX-LABEL: half_imm:
 ; RV64IZHINX:       # %bb.0:
 ; RV64IZHINX-NEXT:    lui a0, 4
 ; RV64IZHINX-NEXT:    addiw a0, a0, 512
+; RV64IZHINX-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINX-NEXT:    ret
 ;
 ; CHECKIZFHMIN-LABEL: half_imm:
@@ -50,12 +52,14 @@ define half @half_imm() nounwind {
 ; RV32IZHINXMIN:       # %bb.0:
 ; RV32IZHINXMIN-NEXT:    lui a0, 4
 ; RV32IZHINXMIN-NEXT:    addi a0, a0, 512
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: half_imm:
 ; RV64IZHINXMIN:       # %bb.0:
 ; RV64IZHINXMIN-NEXT:    lui a0, 4
 ; RV64IZHINXMIN-NEXT:    addiw a0, a0, 512
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   ret half 3.0
 }
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
index 3e0f838270aa5..81e29329e7181 100644
--- a/llvm/test/CodeGen/RISCV/half-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -1797,17 +1797,10 @@ define half @fabs_f16(half %a) nounwind {
 ; CHECKIZFH-NEXT:    fabs.h fa0, fa0
 ; CHECKIZFH-NEXT:    ret
 ;
-; RV32IZHINX-LABEL: fabs_f16:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    slli a0, a0, 17
-; RV32IZHINX-NEXT:    srli a0, a0, 17
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fabs_f16:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    slli a0, a0, 49
-; RV64IZHINX-NEXT:    srli a0, a0, 49
-; RV64IZHINX-NEXT:    ret
+; CHECKIZHINX-LABEL: fabs_f16:
+; CHECKIZHINX:       # %bb.0:
+; CHECKIZHINX-NEXT:    fabs.h a0, a0
+; CHECKIZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fabs_f16:
 ; RV32I:       # %bb.0:
@@ -1839,14 +1832,18 @@ define half @fabs_f16(half %a) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: fabs_f16:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: fabs_f16:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = call half @llvm.fabs.f16(half %a)
   ret half %1
@@ -2094,20 +2091,26 @@ define half @copysign_f16(half %a, half %b) nounwind {
 ;
 ; RV32IZHINXMIN-LABEL: copysign_f16:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV32IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    or a0, a0, a1
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV32IZHINXMIN-NEXT:    ret
 ;
 ; RV64IZHINXMIN-LABEL: copysign_f16:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x11_h killed $x11_h def $x11
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    lui a2, 1048568
 ; RV64IZHINXMIN-NEXT:    and a1, a1, a2
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    or a0, a0, a1
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h killed $x10
 ; RV64IZHINXMIN-NEXT:    ret
   %1 = call half @llvm.copysign.f16(half %a, half %b)
   ret half %1
@@ -2835,6 +2838,7 @@ define i1 @isnan_d_fpclass(half %x) {
 ;
 ; RV32IZHINXMIN-LABEL: isnan_d_fpclass:
 ; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV32IZHINXMIN-NEXT:    slli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    srli a0, a0, 17
 ; RV32IZHINXMIN-NEXT:    li a1, 31
@@ -2844,6 +2848,7 @@ define i1 @isnan_d_fpclass(half %x) {
 ;
 ; RV64IZHINXMIN-LABEL: isnan_d_fpclass:
 ; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    # kill: def $x10_h killed $x10_h def $x10
 ; RV64IZHINXMIN-NEXT:    slli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    srli a0, a0, 49
 ; RV64IZHINXMIN-NEXT:    li a1, 31
diff --git a/llvm/test/CodeGen/RISCV/kcfi-mir.ll b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
index 9d8475e2171ea..e478930d59abc 100644
--- a/llvm/test/CodeGen/RISCV/kcfi-mir.ll
+++ b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
@@ -10,7 +10,7 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 16
   ; CHECK-NEXT:   SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x1, -8
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit-def dead $x1, implicit-def $x2, implicit killed $x10 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_h, implicit-def dead $x1, implicit-def $x2, implicit-def $x2_h, implicit killed $x10 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoCALLIndirect killed $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
   ; CHECK-NEXT:   }
@@ -26,7 +26,7 @@ define void @f2(ptr noundef %x) #0 {
   ; CHECK: bb.0 (%ir-block.0):
   ; CHECK-NEXT:   liveins: $x10
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit killed $x10, implicit $x2 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_h, implicit killed $x10, implicit $x2 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoTAILIndirect killed $x10, implicit $x2
   ; CHECK-NEXT:   }
diff --git a/llvm/test/CodeGen/RISCV/make-compressible-zbc-zhinx.mir b/llvm/test/CodeGen/RISCV/make-compressible-zbc-zhinx.mir
new file mode 100644
index 0000000000000..45fcc792d2fca
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/make-compressible-zbc-zhinx.mir
@@ -0,0 +1,249 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=riscv32 -mattr=+zcb,+zhinx -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+# RUN: llc -o - %s -mtriple=riscv64 -mattr=+zcb,+zhinx -simplify-mir \
+# RUN:   -run-pass=riscv-make-compressible | FileCheck --check-prefixes=CHECK %s
+
+--- |
+  define void @store_common_value_half(ptr %a, ptr %b, ptr %c) #0 {
+  entry:
+    store half 0.0, ptr %a, align 2
+    store half 0.0, ptr %b, align 2
+    store half 0.0, ptr %c, align 2
+    ret void
+  }
+
+  define void @store_common_ptr_half(ptr %p) #0 {
+  entry:
+    store volatile half 2.0, ptr %p, align 2
+    store volatile half 32.0, ptr %p, align 2
+    store volatile half 512.0, ptr %p, align 2
+    ret void
+  }
+
+  define void @load_common_ptr_half(ptr %p) #0 {
+  entry:
+    %0 = load volatile half, ptr %p, align 2
+    %1 = load volatile half, ptr %p, align 2
+    %2 = load volatile half, ptr %p, align 2
+    ret void
+  }
+
+  define void @store_large_offset_half(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds half, ptr %p, i32 100
+    store volatile half 2.0, ptr %0, align 2
+    %1 = getelementptr inbounds half, ptr %p, i32 101
+    store volatile half 32.0, ptr %1, align 2
+    %2 = getelementptr inbounds half, ptr %p, i32 102
+    store volatile half 512.0, ptr %2, align 2
+    %3 = getelementptr inbounds half, ptr %p, i32 103
+    store volatile half 16384.0, ptr %3, align 2
+    ret void
+  }
+
+  define void @load_large_offset_half(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds half, ptr %p, i32 100
+    %a = load volatile half, ptr %0, align 2
+    %1 = getelementptr inbounds half, ptr %p, i32 100
+    %b = load volatile half, ptr %1, align 2
+    %2 = getelementptr inbounds half, ptr %p, i32 101
+    %c = load volatile half, ptr %2, align 2
+    %3 = getelementptr inbounds half, ptr %p, i32 101
+    %d = load volatile half, ptr %3, align 2
+    ret void
+  }
+
+  define void @store_large_offset_no_opt_half(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds i8, ptr %p, i8 100
+    store volatile half 2.0, ptr %0, align 2
+    %1 = getelementptr inbounds i8, ptr %p, i8 101
+    store volatile half 32.0, ptr %1, align 2
+    %2 = getelementptr inbounds i8, ptr %p, i8 104
+    store volatile half 512.0, ptr %2, align 2
+    ret void
+  }
+
+  define void @load_large_offset_no_opt_half(ptr %p) #0 {
+  entry:
+    %0 = getelementptr inbounds half, ptr %p, i32 100
+    %a = load volatile half, ptr %0, align 2
+    %1 = getelementptr inbounds half, ptr %p, i32 101
+    %c = load volatile half, ptr %1, align 2
+    %2 = getelementptr inbounds half, ptr %p, i32 102
+    %d = load volatile half, ptr %2, align 2
+    ret void
+  }
+
+  attributes #0 = { minsize }
+
+...
+---
+name:            store_common_value_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: store_common_value_half
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x13_h = PseudoMV_FPR16INX $x0_h
+    ; CHECK-NEXT: SH_INX $x13_h, killed renamable $x10, 0 :: (store (s16) into %ir.a)
+    ; CHECK-NEXT: SH_INX $x13_h, killed renamable $x11, 0 :: (store (s16) into %ir.b)
+    ; CHECK-NEXT: SH_INX $x13_h, killed renamable $x12, 0 :: (store (s16) into %ir.c)
+    ; CHECK-NEXT: PseudoRET
+    SH_INX $x0_h, killed renamable $x10, 0 :: (store (s16) into %ir.a)
+    SH_INX $x0_h, killed renamable $x11, 0 :: (store (s16) into %ir.b)
+    SH_INX $x0_h, killed renamable $x12, 0 :: (store (s16) into %ir.c)
+    PseudoRET
+
+...
+---
+name:            store_common_ptr_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_common_ptr_half
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x10 = LUI 4
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: SH_INX killed renamable $x10_h, $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = LUI 5
+    ; CHECK-NEXT: SH_INX killed renamable $x10_h, $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: renamable $x10 = LUI 6
+    ; CHECK-NEXT: SH_INX killed renamable $x10_h, killed $x11, 0 :: (volatile store (s16) into %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x10 = LUI 4
+    SH_INX killed renamable $x10_h, renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    renamable $x10 = LUI 5
+    SH_INX killed renamable $x10_h, renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    renamable $x10 = LUI 6
+    SH_INX killed renamable $x10_h, killed renamable $x16, 0 :: (volatile store (s16) into %ir.p)
+    PseudoRET
+
+...
+---
+name:            load_common_ptr_half
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_common_ptr_half
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 0
+    ; CHECK-NEXT: dead $x10_h = LH_INX $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10_h = LH_INX $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: dead $x10_h = LH_INX killed $x11, 0 :: (volatile load (s16) from %ir.p)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10_h = LH_INX renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10_h = LH_INX renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    dead $x10_h = LH_INX killed renamable $x16, 0 :: (volatile load (s16) from %ir.p)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: store_large_offset_half
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = LUI 4
+    ; CHECK-NEXT: $x12 = ADDI $x10, 200
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, $x12, 0 :: (volatile store (s16) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = LUI 5
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, $x12, 0 :: (volatile store (s16) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = LUI 6
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, $x12, 2 :: (volatile store (s16) into %ir.2)
+    ; CHECK-NEXT: renamable $x11 = LUI 7
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, killed $x12, 2 :: (volatile store (s16) into %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = LUI 4
+    SH_INX killed renamable $x11_h, renamable $x10, 200 :: (volatile store (s16) into %ir.0)
+    renamable $x11 = LUI 5
+    SH_INX killed renamable $x11_h, renamable $x10, 200 :: (volatile store (s16) into %ir.1)
+    renamable $x11 = LUI 6
+    SH_INX killed renamable $x11_h, renamable $x10, 202 :: (volatile store (s16) into %ir.2)
+    renamable $x11 = LUI 7
+    SH_INX killed renamable $x11_h, killed renamable $x10, 202 :: (volatile store (s16) into %ir.3)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_half
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x11 = ADDI $x16, 100
+    ; CHECK-NEXT: dead $x10_h = LH_INX $x11, 0 :: (volatile load (s16) from %ir.0)
+    ; CHECK-NEXT: dead $x10_h = LH_INX $x11, 0 :: (volatile load (s16) from %ir.1)
+    ; CHECK-NEXT: dead $x10_h = LH_INX $x11, 2 :: (volatile load (s16) from %ir.2)
+    ; CHECK-NEXT: dead $x10_h = LH_INX killed $x11, 2 :: (volatile load (s16) from %ir.3)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10_h = LH_INX renamable $x16, 100 :: (volatile load (s16) from %ir.0)
+    dead $x10_h = LH_INX renamable $x16, 100 :: (volatile load (s16) from %ir.1)
+    dead $x10_h = LH_INX renamable $x16, 102 :: (volatile load (s16) from %ir.2)
+    dead $x10_h = LH_INX killed renamable $x16, 102 :: (volatile load (s16) from %ir.3)
+    PseudoRET
+
+...
+---
+name:            store_large_offset_no_opt_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: store_large_offset_no_opt_half
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $x11 = LUI 4
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, renamable $x16, 200 :: (volatile store (s16) into %ir.0)
+    ; CHECK-NEXT: renamable $x11 = LUI 5
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, renamable $x16, 202 :: (volatile store (s16) into %ir.1)
+    ; CHECK-NEXT: renamable $x11 = LUI 6
+    ; CHECK-NEXT: SH_INX killed renamable $x11_h, renamable $x16, 204 :: (volatile store (s16) into %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    renamable $x11 = LUI 4
+    SH_INX killed renamable $x11_h, renamable $x16, 200 :: (volatile store (s16) into %ir.0)
+    renamable $x11 = LUI 5
+    SH_INX killed renamable $x11_h, renamable $x16, 202 :: (volatile store (s16) into %ir.1)
+    renamable $x11 = LUI 6
+    SH_INX killed renamable $x11_h, renamable $x16, 204 :: (volatile store (s16) into %ir.2)
+    PseudoRET
+
+...
+---
+name:            load_large_offset_no_opt_half
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x16
+
+    ; CHECK-LABEL: name: load_large_offset_no_opt_half
+    ; CHECK: liveins: $x16
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: dead $x10_h = LH_INX renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    ; CHECK-NEXT: dead $x10_h = LH_INX renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    ; CHECK-NEXT: dead $x10_h = LH_INX killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    ; CHECK-NEXT: PseudoRET
+    dead $x10_h = LH_INX renamable $x16, 100 :: (volatile load (s8) from %ir.0)
+    dead $x10_h = LH_INX renamable $x16, 102 :: (volatile load (s8) from %ir.1)
+    dead $x10_h = LH_INX killed renamable $x16, 104 :: (volatile load (s8) from %ir.2)
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/extractelt-fp.ll b/llvm/test/CodeGen/RISCV/rvv/extractelt-fp.ll
index 209a37bf66ae3..86ef78be97afb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extractelt-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extractelt-fp.ll
@@ -1,197 +1,895 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh,+zvfh,+v -target-abi=ilp32d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v -target-abi=lp64d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,NOZFMIN,ZVFH
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zvfh,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,NOZFMIN,ZVFH
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,NOZFMIN,ZVFHMIN
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,NOZFMIN,ZVFHMIN
+; RUN: llc -mtriple=riscv32 -mattr=+v,+d,+zfhmin,+zfbfmin,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32,ZFMIN
+; RUN: llc -mtriple=riscv64 -mattr=+v,+d,+zfhmin,+zfbfmin,+zvfhmin,+zvfbfmin -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64,ZFMIN
+
+define bfloat @extractelt_nxv1bf16_0(<vscale x 1 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv1bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 1 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv1bf16_imm(<vscale x 1 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv1bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 1 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv1bf16_idx(<vscale x 1 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv1bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 1 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv2bf16_0(<vscale x 2 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv2bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 2 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv2bf16_imm(<vscale x 2 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv2bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 2 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv2bf16_idx(<vscale x 2 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv2bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 2 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv4bf16_0(<vscale x 4 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv4bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 4 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv4bf16_imm(<vscale x 4 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv4bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 4 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv4bf16_idx(<vscale x 4 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv4bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 4 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv8bf16_0(<vscale x 8 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv8bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 8 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv8bf16_imm(<vscale x 8 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv8bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 8 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv8bf16_idx(<vscale x 8 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv8bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 8 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv16bf16_0(<vscale x 16 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv16bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 16 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv16bf16_imm(<vscale x 16 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv16bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 16 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv16bf16_idx(<vscale x 16 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv16bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 16 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv32bf16_0(<vscale x 32 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv32bf16_0:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32bf16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 32 x bfloat> %v, i32 0
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv32bf16_imm(<vscale x 32 x bfloat> %v) {
+; NOZFMIN-LABEL: extractelt_nxv32bf16_imm:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32bf16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 32 x bfloat> %v, i32 2
+  ret bfloat %r
+}
+
+define bfloat @extractelt_nxv32bf16_idx(<vscale x 32 x bfloat> %v, i32 zeroext %idx) {
+; NOZFMIN-LABEL: extractelt_nxv32bf16_idx:
+; NOZFMIN:       # %bb.0:
+; NOZFMIN-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
+; NOZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; NOZFMIN-NEXT:    vmv.x.s a0, v8
+; NOZFMIN-NEXT:    lui a1, 1048560
+; NOZFMIN-NEXT:    or a0, a0, a1
+; NOZFMIN-NEXT:    fmv.w.x fa0, a0
+; NOZFMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32bf16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
+  %r = extractelement <vscale x 32 x bfloat> %v, i32 %idx
+  ret bfloat %r
+}
 
 define half @extractelt_nxv1f16_0(<vscale x 1 x half> %v) {
-; CHECK-LABEL: extractelt_nxv1f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv1f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv1f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 1 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv1f16_imm(<vscale x 1 x half> %v) {
-; CHECK-LABEL: extractelt_nxv1f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv1f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv1f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 1 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv1f16_idx(<vscale x 1 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv1f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv1f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv1f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv1f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 1 x half> %v, i32 %idx
   ret half %r
 }
 
 define half @extractelt_nxv2f16_0(<vscale x 2 x half> %v) {
-; CHECK-LABEL: extractelt_nxv2f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv2f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv2f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 2 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv2f16_imm(<vscale x 2 x half> %v) {
-; CHECK-LABEL: extractelt_nxv2f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv2f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv2f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 2 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv2f16_idx(<vscale x 2 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv2f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv2f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv2f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv2f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, mf2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 2 x half> %v, i32 %idx
   ret half %r
 }
 
 define half @extractelt_nxv4f16_0(<vscale x 4 x half> %v) {
-; CHECK-LABEL: extractelt_nxv4f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv4f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv4f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 4 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv4f16_imm(<vscale x 4 x half> %v) {
-; CHECK-LABEL: extractelt_nxv4f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv4f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv4f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 4 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv4f16_idx(<vscale x 4 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv4f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv4f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv4f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv4f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 4 x half> %v, i32 %idx
   ret half %r
 }
 
 define half @extractelt_nxv8f16_0(<vscale x 8 x half> %v) {
-; CHECK-LABEL: extractelt_nxv8f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv8f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv8f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 8 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv8f16_imm(<vscale x 8 x half> %v) {
-; CHECK-LABEL: extractelt_nxv8f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv8f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv8f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 8 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv8f16_idx(<vscale x 8 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv8f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv8f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv8f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv8f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m2, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 8 x half> %v, i32 %idx
   ret half %r
 }
 
 define half @extractelt_nxv16f16_0(<vscale x 16 x half> %v) {
-; CHECK-LABEL: extractelt_nxv16f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv16f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv16f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 16 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv16f16_imm(<vscale x 16 x half> %v) {
-; CHECK-LABEL: extractelt_nxv16f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv16f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv16f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 16 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv16f16_idx(<vscale x 16 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv16f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv16f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv16f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv16f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m4, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 16 x half> %v, i32 %idx
   ret half %r
 }
 
 define half @extractelt_nxv32f16_0(<vscale x 32 x half> %v) {
-; CHECK-LABEL: extractelt_nxv32f16_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv32f16_0:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv32f16_0:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32f16_0:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 32 x half> %v, i32 0
   ret half %r
 }
 
 define half @extractelt_nxv32f16_imm(<vscale x 32 x half> %v) {
-; CHECK-LABEL: extractelt_nxv32f16_imm:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv32f16_imm:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv32f16_imm:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32f16_imm:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; ZFMIN-NEXT:    vslidedown.vi v8, v8, 2
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 32 x half> %v, i32 2
   ret half %r
 }
 
 define half @extractelt_nxv32f16_idx(<vscale x 32 x half> %v, i32 zeroext %idx) {
-; CHECK-LABEL: extractelt_nxv32f16_idx:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vfmv.f.s fa0, v8
-; CHECK-NEXT:    ret
+; ZVFH-LABEL: extractelt_nxv32f16_idx:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
+; ZVFH-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFH-NEXT:    vfmv.f.s fa0, v8
+; ZVFH-NEXT:    ret
+;
+; ZVFHMIN-LABEL: extractelt_nxv32f16_idx:
+; ZVFHMIN:       # %bb.0:
+; ZVFHMIN-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZVFHMIN-NEXT:    vmv.x.s a0, v8
+; ZVFHMIN-NEXT:    lui a1, 1048560
+; ZVFHMIN-NEXT:    or a0, a0, a1
+; ZVFHMIN-NEXT:    fmv.w.x fa0, a0
+; ZVFHMIN-NEXT:    ret
+;
+; ZFMIN-LABEL: extractelt_nxv32f16_idx:
+; ZFMIN:       # %bb.0:
+; ZFMIN-NEXT:    vsetivli zero, 1, e16, m8, ta, ma
+; ZFMIN-NEXT:    vslidedown.vx v8, v8, a0
+; ZFMIN-NEXT:    vmv.x.s a0, v8
+; ZFMIN-NEXT:    fmv.h.x fa0, a0
+; ZFMIN-NEXT:    ret
   %r = extractelement <vscale x 32 x half> %v, i32 %idx
   ret half %r
 }
@@ -636,10 +1334,10 @@ define double @extractelt_nxv16f64_neg1(<vscale x 16 x double> %v) {
 ; RV64-NEXT:    slli a2, a2, 1
 ; RV64-NEXT:    addi a2, a2, -1
 ; RV64-NEXT:    vs8r.v v16, (a3)
-; RV64-NEXT:    bltu a2, a1, .LBB52_2
+; RV64-NEXT:    bltu a2, a1, .LBB70_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a2, a1
-; RV64-NEXT:  .LBB52_2:
+; RV64-NEXT:  .LBB70_2:
 ; RV64-NEXT:    slli a2, a2, 3
 ; RV64-NEXT:    add a0, a0, a2
 ; RV64-NEXT:    fld fa0, 0(a0)
@@ -669,10 +1367,10 @@ define double @extractelt_nxv16f64_idx(<vscale x 16 x double> %v, i32 zeroext %i
 ; RV32-NEXT:    csrr a1, vlenb
 ; RV32-NEXT:    slli a2, a1, 1
 ; RV32-NEXT:    addi a2, a2, -1
-; RV32-NEXT:    bltu a0, a2, .LBB54_2
+; RV32-NEXT:    bltu a0, a2, .LBB72_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    mv a0, a2
-; RV32-NEXT:  .LBB54_2:
+; RV32-NEXT:  .LBB72_2:
 ; RV32-NEXT:    addi sp, sp, -80
 ; RV32-NEXT:    .cfi_def_cfa_offset 80
 ; RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
@@ -704,10 +1402,10 @@ define double @extractelt_nxv16f64_idx(<vscale x 16 x double> %v, i32 zeroext %i
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    slli a2, a1, 1
 ; RV64-NEXT:    addi a2, a2, -1
-; RV64-NEXT:    bltu a0, a2, .LBB54_2
+; RV64-NEXT:    bltu a0, a2, .LBB72_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a0, a2
-; RV64-NEXT:  .LBB54_2:
+; RV64-NEXT:  .LBB72_2:
 ; RV64-NEXT:    addi sp, sp, -80
 ; RV64-NEXT:    .cfi_def_cfa_offset 80
 ; RV64-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 1395dc914bb40..3c184c112e77a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -2261,10 +2261,7 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    fld fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    li a2, -1
 ; CHECK-V-NEXT:    srli a3, a2, 1
@@ -2394,10 +2391,7 @@ define <2 x i64> @utest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    call __fixunsdfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    fld fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixunsdfti
 ; CHECK-V-NEXT:    snez a1, a1
 ; CHECK-V-NEXT:    snez a2, s1
@@ -2506,10 +2500,7 @@ define <2 x i64> @ustest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    fld fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv a2, s1
 ; CHECK-V-NEXT:    blez s1, .LBB20_2
@@ -2668,10 +2659,7 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    flw fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    li a2, -1
 ; CHECK-V-NEXT:    srli a3, a2, 1
@@ -2801,10 +2789,7 @@ define <2 x i64> @utest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    call __fixunssfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    flw fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixunssfti
 ; CHECK-V-NEXT:    snez a1, a1
 ; CHECK-V-NEXT:    snez a2, s1
@@ -2913,10 +2898,7 @@ define <2 x i64> @ustest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    flw fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv a2, s1
 ; CHECK-V-NEXT:    blez s1, .LBB23_2
@@ -5597,10 +5579,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    fld fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    li a2, -1
 ; CHECK-V-NEXT:    srli a3, a2, 1
@@ -5831,10 +5810,7 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    fld fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixdfti
 ; CHECK-V-NEXT:    mv a2, a1
 ; CHECK-V-NEXT:    blez a1, .LBB47_2
@@ -5983,10 +5959,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    flw fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    li a2, -1
 ; CHECK-V-NEXT:    srli a3, a2, 1
@@ -6217,10 +6190,7 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv s0, a0
 ; CHECK-V-NEXT:    mv s1, a1
-; CHECK-V-NEXT:    addi a0, sp, 32
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-V-NEXT:    vfmv.f.s fa0, v8
+; CHECK-V-NEXT:    flw fa0, 32(sp) # 8-byte Folded Reload
 ; CHECK-V-NEXT:    call __fixsfti
 ; CHECK-V-NEXT:    mv a2, a1
 ; CHECK-V-NEXT:    blez a1, .LBB50_2
diff --git a/llvm/test/CodeGen/RISCV/rvv/stack-folding.ll b/llvm/test/CodeGen/RISCV/rvv/stack-folding.ll
index 4771d7fe6ec92..f966835622a9f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/stack-folding.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/stack-folding.ll
@@ -160,3 +160,100 @@ truebb:
 falsebb:
   ret i8 0
 }
+
+define double @f64(<vscale x 1 x double> %v, i1 %c) {
+; RV32-LABEL: f64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 1
+; RV32-NEXT:    sub sp, sp, a1
+; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; RV32-NEXT:    addi a1, sp, 16
+; RV32-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    andi a0, a0, 1
+; RV32-NEXT:    #APP
+; RV32-NEXT:    #NO_APP
+; RV32-NEXT:    beqz a0, .LBB4_2
+; RV32-NEXT:  # %bb.1: # %truebb
+; RV32-NEXT:    fld fa0, 16(sp) # 8-byte Folded Reload
+; RV32-NEXT:    j .LBB4_3
+; RV32-NEXT:  .LBB4_2: # %falsebb
+; RV32-NEXT:    fcvt.d.w fa0, zero
+; RV32-NEXT:  .LBB4_3: # %falsebb
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    slli a0, a0, 1
+; RV32-NEXT:    add sp, sp, a0
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: f64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 1
+; RV64-NEXT:    sub sp, sp, a1
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
+; RV64-NEXT:    andi a0, a0, 1
+; RV64-NEXT:    #APP
+; RV64-NEXT:    #NO_APP
+; RV64-NEXT:    beqz a0, .LBB4_2
+; RV64-NEXT:  # %bb.1: # %truebb
+; RV64-NEXT:    fld fa0, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT:    j .LBB4_3
+; RV64-NEXT:  .LBB4_2: # %falsebb
+; RV64-NEXT:    fmv.d.x fa0, zero
+; RV64-NEXT:  .LBB4_3: # %falsebb
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 1
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  tail call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  br i1 %c, label %truebb, label %falsebb
+truebb:
+  %x = extractelement <vscale x 1 x double> %v, i32 0
+  ret double %x
+falsebb:
+  ret double 0.0
+}
+
+define float @f32(<vscale x 2 x float> %v, i1 %c) {
+; CHECK-LABEL: f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 1
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x02, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 2 * vlenb
+; CHECK-NEXT:    addi a1, sp, 16
+; CHECK-NEXT:    vs1r.v v8, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    andi a0, a0, 1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    beqz a0, .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %truebb
+; CHECK-NEXT:    flw fa0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    j .LBB5_3
+; CHECK-NEXT:  .LBB5_2: # %falsebb
+; CHECK-NEXT:    fmv.w.x fa0, zero
+; CHECK-NEXT:  .LBB5_3: # %falsebb
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+  tail call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31}"()
+  br i1 %c, label %truebb, label %falsebb
+truebb:
+  %x = extractelement <vscale x 2 x float> %v, i32 0
+  ret float %x
+falsebb:
+  ret float 0.0
+}
+
diff --git a/llvm/test/CodeGen/RISCV/tail-calls.ll b/llvm/test/CodeGen/RISCV/tail-calls.ll
index d3e495bb723ad..4dd6ed68ff981 100644
--- a/llvm/test/CodeGen/RISCV/tail-calls.ll
+++ b/llvm/test/CodeGen/RISCV/tail-calls.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple riscv32-unknown-linux-gnu -o - %s | FileCheck %s
+; RUN: llc -mtriple riscv32-unknown-linux-gnu -mattr=experimental-zicfilp \
+; RUN:   -code-model=large -o - %s \
+; RUN:   | FileCheck %s -check-prefix=CHECK-LARGE-ZICFILP
 ; RUN: llc -mtriple riscv32-unknown-elf       -o - %s | FileCheck %s
 
 ; Perform tail call optimization for global address.
@@ -8,6 +11,14 @@ define i32 @caller_tail(i32 %i) nounwind {
 ; CHECK-LABEL: caller_tail:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    tail callee_tail
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_tail:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi0:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI0_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi0)(a1)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
 entry:
   %r = tail call i32 @callee_tail(i32 %i)
   ret i32 %r
@@ -26,6 +37,21 @@ define void @caller_extern(ptr %src) optsize {
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    mv a1, a3
 ; CHECK-NEXT:    tail memcpy
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_extern:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi1:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI1_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw a1, %pcrel_lo(.Lpcrel_hi1)(a1)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi2:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a2, %pcrel_hi(.LCPI1_1)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi2)(a2)
+; CHECK-LARGE-ZICFILP-NEXT:    li a2, 7
+; CHECK-LARGE-ZICFILP-NEXT:    mv a3, a0
+; CHECK-LARGE-ZICFILP-NEXT:    mv a0, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a1, a3
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
 entry:
   tail call void @llvm.memcpy.p0.p0.i32(ptr @dest, ptr %src, i32 7, i1 false)
   ret void
@@ -43,6 +69,21 @@ define void @caller_extern_pgso(ptr %src) !prof !14 {
 ; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    mv a1, a3
 ; CHECK-NEXT:    tail memcpy
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_extern_pgso:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi3:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI2_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw a1, %pcrel_lo(.Lpcrel_hi3)(a1)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi4:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a2, %pcrel_hi(.LCPI2_1)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi4)(a2)
+; CHECK-LARGE-ZICFILP-NEXT:    li a2, 7
+; CHECK-LARGE-ZICFILP-NEXT:    mv a3, a0
+; CHECK-LARGE-ZICFILP-NEXT:    mv a0, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a1, a3
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
 entry:
   tail call void @llvm.memcpy.p0.p0.i32(ptr @dest_pgso, ptr %src, i32 7, i1 false)
   ret void
@@ -63,8 +104,21 @@ define void @caller_indirect_tail(i32 %a) nounwind {
 ; CHECK-NEXT:    lui t1, %hi(callee_indirect1)
 ; CHECK-NEXT:    addi t1, t1, %lo(callee_indirect1)
 ; CHECK-NEXT:    jr t1
-
-
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_indirect_tail:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    beqz a0, .LBB3_2
+; CHECK-LARGE-ZICFILP-NEXT:  # %bb.1: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi6:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI3_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t1, %pcrel_lo(.Lpcrel_hi6)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t1
+; CHECK-LARGE-ZICFILP-NEXT:  .LBB3_2:
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi5:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI3_1)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t1, %pcrel_lo(.Lpcrel_hi5)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t1
 entry:
   %tobool = icmp eq i32 %a, 0
   %callee = select i1 %tobool, ptr @callee_indirect1, ptr @callee_indirect2
@@ -86,6 +140,19 @@ define i32 @caller_indirect_no_t0(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5
 ; CHECK-NEXT:    mv a5, a6
 ; CHECK-NEXT:    mv a6, a7
 ; CHECK-NEXT:    jr t1
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_indirect_no_t0:
+; CHECK-LARGE-ZICFILP:       # %bb.0:
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    mv t1, a0
+; CHECK-LARGE-ZICFILP-NEXT:    mv a0, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a1, a2
+; CHECK-LARGE-ZICFILP-NEXT:    mv a2, a3
+; CHECK-LARGE-ZICFILP-NEXT:    mv a3, a4
+; CHECK-LARGE-ZICFILP-NEXT:    mv a4, a5
+; CHECK-LARGE-ZICFILP-NEXT:    mv a5, a6
+; CHECK-LARGE-ZICFILP-NEXT:    mv a6, a7
+; CHECK-LARGE-ZICFILP-NEXT:    jr t1
   %9 = tail call i32 %0(i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7)
   ret i32 %9
 }
@@ -108,6 +175,26 @@ define void @caller_varargs(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_varargs:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi7:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a2, %pcrel_hi(.LCPI5_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi7)(a2)
+; CHECK-LARGE-ZICFILP-NEXT:    sw a0, 0(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    mv a2, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a3, a0
+; CHECK-LARGE-ZICFILP-NEXT:    mv a4, a0
+; CHECK-LARGE-ZICFILP-NEXT:    mv a5, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a6, a1
+; CHECK-LARGE-ZICFILP-NEXT:    mv a7, a0
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   %call = tail call i32 (i32, ...) @callee_varargs(i32 %a, i32 %b, i32 %b, i32 %a, i32 %a, i32 %b, i32 %b, i32 %a, i32 %a)
   ret void
@@ -136,6 +223,31 @@ define i32 @caller_args(i32 %a, i32 %b, i32 %c, i32 %dd, i32 %e, i32 %ff, i32 %g
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_args:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -32
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    lw t0, 32(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t1, 36(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t3, 40(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t4, 44(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, 48(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t5, 52(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t5, 20(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t2, 16(sp)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi8:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc t2, %pcrel_hi(.LCPI6_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi8)(t2)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t4, 12(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t3, 8(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t1, 4(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw t0, 0(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   %r = tail call i32 @callee_args(i32 %a, i32 %b, i32 %c, i32 %dd, i32 %e, i32 %ff, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n)
   ret i32 %r
@@ -158,6 +270,25 @@ define void @caller_indirect_args() nounwind {
 ; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_indirect_args:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -32
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    lui a0, 262128
+; CHECK-LARGE-ZICFILP-NEXT:    sw a0, 12(sp)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi9:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI7_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi9)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 8(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 4(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    mv a0, sp
+; CHECK-LARGE-ZICFILP-NEXT:    sw zero, 0(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 32
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   %call = tail call i32 @callee_indirect_args(fp128 0xL00000000000000003FFF000000000000)
   ret void
@@ -169,6 +300,14 @@ define void @caller_weak() nounwind {
 ; CHECK-LABEL: caller_weak:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    tail callee_weak
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_weak:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi10:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI8_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi10)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
 entry:
   tail call void @callee_weak()
   ret void
@@ -217,6 +356,48 @@ define void @caller_irq() nounwind "interrupt"="machine" {
 ; CHECK-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    mret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_irq:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -64
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi11:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI9_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi11)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 64
+; CHECK-LARGE-ZICFILP-NEXT:    mret
 entry:
   tail call void @callee_irq()
   ret void
@@ -238,6 +419,22 @@ define i32 @caller_byval() nounwind {
 ; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_byval:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:    lw a0, 8(sp)
+; CHECK-LARGE-ZICFILP-NEXT:    sw a0, 4(sp)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi12:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI10_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi12)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    addi a0, sp, 4
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   %a = alloca ptr
   %r = tail call i32 @callee_byval(ptr byval(ptr) %a)
@@ -260,6 +457,22 @@ define void @caller_nostruct() nounwind {
 ; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_nostruct:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi13:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI11_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw a0, %pcrel_lo(.Lpcrel_hi13)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi14:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI11_1)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi14)(a1)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   tail call void @callee_struct(ptr sret(%struct.A) @a)
   ret void
@@ -276,6 +489,19 @@ define void @caller_struct(ptr sret(%struct.A) %a) nounwind {
 ; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: caller_struct:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi15:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI12_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi15)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   tail call void @callee_nostruct()
   ret void
@@ -291,6 +517,19 @@ define i32 @disable_tail_calls(i32 %i) nounwind "disable-tail-calls"="true" {
 ; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
+;
+; CHECK-LARGE-ZICFILP-LABEL: disable_tail_calls:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, -16
+; CHECK-LARGE-ZICFILP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi16:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a1, %pcrel_hi(.LCPI13_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi16)(a1)
+; CHECK-LARGE-ZICFILP-NEXT:    jalr t2
+; CHECK-LARGE-ZICFILP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK-LARGE-ZICFILP-NEXT:    addi sp, sp, 16
+; CHECK-LARGE-ZICFILP-NEXT:    ret
 entry:
   %rv = tail call i32 @callee_tail(i32 %i)
   ret i32 %rv
@@ -317,6 +556,35 @@ define i32 @duplicate_returns(i32 %a, i32 %b) nounwind {
 ; CHECK-NEXT:    tail test1
 ; CHECK-NEXT:  .LBB14_6: # %if.else8
 ; CHECK-NEXT:    tail test3
+;
+; CHECK-LARGE-ZICFILP-LABEL: duplicate_returns:
+; CHECK-LARGE-ZICFILP:       # %bb.0: # %entry
+; CHECK-LARGE-ZICFILP-NEXT:    lpad 0
+; CHECK-LARGE-ZICFILP-NEXT:    beqz a0, .LBB14_4
+; CHECK-LARGE-ZICFILP-NEXT:  # %bb.1: # %if.else
+; CHECK-LARGE-ZICFILP-NEXT:    beqz a1, .LBB14_5
+; CHECK-LARGE-ZICFILP-NEXT:  # %bb.2: # %if.else4
+; CHECK-LARGE-ZICFILP-NEXT:    bge a1, a0, .LBB14_6
+; CHECK-LARGE-ZICFILP-NEXT:  # %bb.3: # %if.then6
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi19:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI14_1)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi19)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
+; CHECK-LARGE-ZICFILP-NEXT:  .LBB14_4: # %if.then
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi17:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI14_3)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi17)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
+; CHECK-LARGE-ZICFILP-NEXT:  .LBB14_5: # %if.then2
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi18:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI14_2)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi18)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
+; CHECK-LARGE-ZICFILP-NEXT:  .LBB14_6: # %if.else8
+; CHECK-LARGE-ZICFILP-NEXT:  .Lpcrel_hi20:
+; CHECK-LARGE-ZICFILP-NEXT:    auipc a0, %pcrel_hi(.LCPI14_0)
+; CHECK-LARGE-ZICFILP-NEXT:    lw t2, %pcrel_lo(.Lpcrel_hi20)(a0)
+; CHECK-LARGE-ZICFILP-NEXT:    jr t2
 entry:
   %cmp = icmp eq i32 %a, 0
   br i1 %cmp, label %if.then, label %if.else
diff --git a/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll b/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
index 794dcd6d9f3fb..2cf55f662df02 100644
--- a/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
+++ b/llvm/test/CodeGen/SPIRV/debug-info/debug-compilation-unit.ll
@@ -6,21 +6,29 @@
 ; CHECK-MIR-DAG: [[type_void:%[0-9]+\:type]] = OpTypeVoid
 ; CHECK-MIR-DAG: [[type_i64:%[0-9]+\:type\(s64\)]] = OpTypeInt 32, 0
 ; CHECK-MIR-DAG: [[dwarf_version:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 5
-; CHECK-MIR-DAG: [[source_language:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 3
-; CHECK-MIR-DAG: [[debug_info_version:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 21
-; CHECK-MIR-DAG: [[filename_str:%[0-9]+\:id\(s32\)]] = OpString 1094795567, 1094795585, 792805697, 1111638594, 1111638594, 1128481583, 1128481603, {{1697596227|1700545347}}, 1886216568, 1663985004, 0
-; CHECK-MIR-DAG: [[debug_source:%[0-9]+\:id\(s32\)]] = OpExtInst [[type_void]], 3, 35, [[filename_str]]
-; CHECK-MIR-DAG: [[debug_compilation_unit:%[0-9]+\:id\(s32\)]] = OpExtInst [[type_void]], 3, 1, [[source_language]], [[dwarf_version]], [[debug_source]], [[debug_info_version]]
+; CHECK-MIR-DAG: [[debug_info_version:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 3
+; CHECK-MIR-DAG: [[source_language_sycl:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 7
+; CHECK-MIR-DAG: [[source_language_cpp:%[0-9]+\:iid\(s32\)]] = OpConstantI [[type_i64]], 4
+; CHECK-MIR-DAG: [[filename_str_sycl:%[0-9]+\:id\(s32\)]] = OpString 1094795567, 1094795585, 792805697, 1111638594, 1111638594, 1128481583, 1128481603, {{1697596227|1700545347}}, 1886216568, 1663985004, 0
+; CHECK-MIR-DAG: [[filename_str_cpp:%[0-9]+\:id\(s32\)]] = OpString 1145324591, 1145324612, 793003076, 1162167621, 1162167621, 1179010607, 1179010630, 1697596998, 1886216568, 774989164, 7368803
+; CHECK-MIR-DAG: [[debug_source_sycl:%[0-9]+\:id\(s32\)]] = OpExtInst [[type_void]], 3, 35, [[filename_str_sycl]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 1, [[debug_info_version]], [[dwarf_version]], [[debug_source_sycl]], [[source_language_sycl]]
+; CHECK-MIR-DAG: [[debug_source_cpp:%[0-9]+\:id\(s32\)]] = OpExtInst [[type_void]], 3, 35, [[filename_str_cpp]]
+; CHECK-MIR-DAG: OpExtInst [[type_void]], 3, 1, [[debug_info_version]], [[dwarf_version]], [[debug_source_cpp]], [[source_language_cpp]]
 
 ; CHECK-SPIRV: [[ext_inst_non_semantic:%[0-9]+]] = OpExtInstImport "NonSemantic.Shader.DebugInfo.100"
-; CHECK-SPIRV: [[filename_str:%[0-9]+]] = OpString "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC{{[/\\]}}example.c"
+; CHECK-SPIRV: [[filename_str_sycl:%[0-9]+]] = OpString "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC{{[/\\]}}example.c"
+; CHECK-SPIRV: [[filename_str_cpp:%[0-9]+]] = OpString "/DDDDDDDDDD/EEEEEEEE/FFFFFFFFF{{[/\\]}}example1.cpp"
 ; CHECK-SPIRV-DAG: [[type_void:%[0-9]+]] = OpTypeVoid
 ; CHECK-SPIRV-DAG: [[type_i32:%[0-9]+]] = OpTypeInt 32 0
 ; CHECK-SPIRV-DAG: [[dwarf_version:%[0-9]+]] = OpConstant [[type_i32]] 5
-; CHECK-SPIRV-DAG: [[debug_info_version:%[0-9]+]] = OpConstant [[type_i32]] 21
-; CHECK-SPIRV-DAG: [[source_language:%[0-9]+]] = OpConstant [[type_i32]] 3
-; CHECK-SPIRV: [[debug_source:%[0-9]+]] = OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugSource [[filename_str]]
-; CHECK-SPIRV: [[debug_compiation_unit:%[0-9]+]] = OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugCompilationUnit [[source_language]] [[dwarf_version]] [[debug_source]] [[debug_info_version]]
+; CHECK-SPIRV-DAG: [[source_language_sycl:%[0-9]+]] = OpConstant [[type_i32]] 7
+; CHECK-SPIRV-DAG: [[source_language_cpp:%[0-9]+]] = OpConstant [[type_i32]] 4
+; CHECK-SPIRV-DAG: [[debug_info_version:%[0-9]+]] = OpConstant [[type_i32]] 3
+; CHECK-SPIRV: [[debug_source_sycl:%[0-9]+]] = OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugSource [[filename_str_sycl]]
+; CHECK-SPIRV: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugCompilationUnit [[debug_info_version]] [[dwarf_version]] [[debug_source_sycl]] [[source_language_sycl]]
+; CHECK-SPIRV: [[debug_source_cpp:%[0-9]+]] = OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugSource [[filename_str_cpp]]
+; CHECK-SPIRV: OpExtInst [[type_void]] [[ext_inst_non_semantic]] DebugCompilationUnit [[debug_info_version]] [[dwarf_version]] [[debug_source_cpp]] [[source_language_cpp]]
 
 ; CHECK-OPTION-NOT: OpExtInstImport "NonSemantic.Shader.DebugInfo.100"
 ; CHECK-OPTION-NOT: OpString "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC{{[/\\]}}example.c"
@@ -37,12 +45,14 @@ entry:
 }
 ; CHECK-SPIRV-NOT: Lfunc_end1:
 
-!llvm.dbg.cu = !{!0}
+!llvm.dbg.cu = !{!0, !6}
 !llvm.module.flags = !{!2, !3, !4, !5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_OpenCL, file: !1, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_SYCL, file: !1, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "example.c", directory: "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC", checksumkind: CSK_MD5, checksum: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !{i32 1, !"wchar_size", i32 4}
 !5 = !{i32 7, !"frame-pointer", i32 2}
+!6 = distinct !DICompileUnit(language: DW_LANG_OpenCL_CPP, file: !7, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!7 = !DIFile(filename: "example1.cpp", directory: "/DDDDDDDDDD/EEEEEEEE/FFFFFFFFF", checksumkind: CSK_MD5, checksum: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
diff --git a/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
index e3c382d01c6c0..d12914d378542 100644
--- a/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
+++ b/llvm/test/CodeGen/SPIRV/debug-info/debug-type-basic.ll
@@ -163,7 +163,7 @@ entry:
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3, !4, !5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_Zig, file: !1, producer: "clang version XX.X.XXXX (FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
 !1 = !DIFile(filename: "example.cpp", directory: "/AAAAAAAAAA/BBBBBBBB/CCCCCCCCC", checksumkind: CSK_MD5, checksum: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF")
 !2 = !{i32 7, !"Dwarf Version", i32 5}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
new file mode 100644
index 0000000000000..bdbfc133efa29
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/atan2.ll
@@ -0,0 +1,49 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[#op_ext_glsl:]] = OpExtInstImport "GLSL.std.450"
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+
+define noundef float @atan2_float(float noundef %a, float noundef %b) {
+entry:
+; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#]] = OpExtInst %[[#float_32]] %[[#op_ext_glsl]] Atan2 %[[#arg0]] %[[#arg1]]
+  %elt.atan2 = call float @llvm.atan2.f32(float %a, float %b)
+  ret float %elt.atan2
+}
+
+define noundef half @atan2_half(half noundef %a, half noundef %b) {
+entry:
+; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#]]
+; CHECK: %[[#]] = OpExtInst %[[#float_16]] %[[#op_ext_glsl]] Atan2 %[[#arg0]] %[[#arg1]]
+  %elt.atan2 = call half @llvm.atan2.f16(half %a, half %b)
+  ret half %elt.atan2
+}
+
+define noundef <4 x float> @atan2_float4(<4 x float> noundef %a, <4 x float> noundef %b) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec4_float_32]] %[[#op_ext_glsl]] Atan2 %[[#arg0]] %[[#arg1]]
+  %elt.atan2 = call <4 x float> @llvm.atan2.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %elt.atan2
+}
+
+define noundef <4 x half> @atan2_half4(<4 x half> noundef %a, <4 x half> noundef %b) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpExtInst %[[#vec4_float_16]] %[[#op_ext_glsl]] Atan2 %[[#arg0]] %[[#arg1]]
+  %elt.atan2 = call <4 x half> @llvm.atan2.v4f16(<4 x half> %a, <4 x half> %b)
+  ret <4 x half> %elt.atan2
+}
+
+declare half @llvm.atan2.f16(half, half)
+declare float @llvm.atan2.f32(float, float)
+declare <4 x half> @llvm.atan2.v4f16(<4 x half>, <4 x half>)
+declare <4 x float> @llvm.atan2.v4f32(<4 x float>, <4 x float>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll
new file mode 100644
index 0000000000000..2281ccf52bbb4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/smul.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpSMulExtended %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i8, i1 } @llvm.smul.with.overflow.i8(i8 %a, i8 %b)
+  %cmp = extractvalue { i8, i1 } %umul, 1
+  %umul.value = extractvalue { i8, i1 } %umul, 0
+  %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+  store i8 %storemerge, ptr %c, align 1
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpSMulExtended %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i32, i1 } @llvm.smul.with.overflow.i32(i32 %b, i32 %a)
+  %umul.val = extractvalue { i32, i1 } %umul, 0
+  %umul.ov = extractvalue { i32, i1 } %umul, 1
+  %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+  store i32 %spec.select, ptr %c, align 4
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpSMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+  %umul = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+  %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+  %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+  %zero = alloca <2 x i64>, align 16
+  %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+  store <2 x i64> %spec.select, ptr %p
+  ret void
+}
+
+declare {i8, i1} @llvm.smul.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
new file mode 100644
index 0000000000000..cecd6f60655dc
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpIAddCarry %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 %a, i8 %b)
+  %cmp = extractvalue { i8, i1 } %umul, 1
+  %umul.value = extractvalue { i8, i1 } %umul, 0
+  %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+  store i8 %storemerge, ptr %c, align 1
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpIAddCarry %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %b, i32 %a)
+  %umul.val = extractvalue { i32, i1 } %umul, 0
+  %umul.ov = extractvalue { i32, i1 } %umul, 1
+  %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+  store i32 %spec.select, ptr %c, align 4
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpIAddCarry %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+  %umul = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+  %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+  %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+  %zero = alloca <2 x i64>, align 16
+  %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+  store <2 x i64> %spec.select, ptr %p
+  ret void
+}
+
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
index c34771bf381ea..7113dd692f6ac 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/umul.with.overflow.ll
@@ -1,54 +1,89 @@
-; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_8:]] "spirv.llvm_umul_with_overflow_i8"
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_32:]] "spirv.llvm_umul_with_overflow_i32"
-; CHECK-SPIRV: OpName %[[#NAME_UMUL_FUNC_VEC_I64:]] "spirv.llvm_umul_with_overflow_v2i64"
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
-define dso_local spir_func void @_Z4foo8hhPh(i8 zeroext %a, i8 zeroext %b, i8* nocapture %c) local_unnamed_addr {
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpUMulExtended %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
 entry:
-  ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_8]]
   %umul = tail call { i8, i1 } @llvm.umul.with.overflow.i8(i8 %a, i8 %b)
   %cmp = extractvalue { i8, i1 } %umul, 1
   %umul.value = extractvalue { i8, i1 } %umul, 0
   %storemerge = select i1 %cmp, i8 0, i8 %umul.value
-  store i8 %storemerge, i8* %c, align 1
+  store i8 %storemerge, ptr %c, align 1
   ret void
 }
 
-define dso_local spir_func void @_Z5foo32jjPj(i32 %a, i32 %b, i32* nocapture %c) local_unnamed_addr {
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpUMulExtended %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
 entry:
-  ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_32]]
   %umul = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %b, i32 %a)
   %umul.val = extractvalue { i32, i1 } %umul, 0
   %umul.ov = extractvalue { i32, i1 } %umul, 1
   %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
-  store i32 %spec.select, i32* %c, align 4
+  store i32 %spec.select, ptr %c, align 4
   ret void
 }
 
-define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64>* %p) nounwind {
-  ; CHECK-SPIRV: %[[#]] = OpFunctionCall %[[#]] %[[#NAME_UMUL_FUNC_VEC_I64]]
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpUMulExtended %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
   %umul = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
   %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
   %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
   %zero = alloca <2 x i64>, align 16
   %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
-  store <2 x i64> %spec.select, <2 x i64>* %p
+  store <2 x i64> %spec.select, ptr %p
   ret void
 }
 
-; CHECK-SPIRV: %[[#NAME_UMUL_FUNC_8]] = OpFunction %[[#]]
-; CHECK-SPIRV: %[[#VAR_A:]] = OpFunctionParameter %[[#]]
-; CHECK-SPIRV: %[[#VAR_B:]] = OpFunctionParameter %[[#]]
-; CHECK-SPIRV: %[[#MUL_RES:]] = OpIMul %[[#]] %[[#VAR_A]] %[[#VAR_B]]
-; CHECK-SPIRV: %[[#DIV_RES:]] = OpUDiv %[[#]] %[[#MUL_RES]] %[[#VAR_A]]
-; CHECK-SPIRV: %[[#CMP_RES:]] = OpINotEqual %[[#]] %[[#VAR_A]] %[[#DIV_RES]]
-; CHECK-SPIRV: %[[#INSERT_RES:]] = OpCompositeInsert %[[#]] %[[#MUL_RES]]
-; CHECK-SPIRV: %[[#INSERT_RES_1:]] = OpCompositeInsert %[[#]] %[[#CMP_RES]] %[[#INSERT_RES]]
-; CHECK-SPIRV: OpReturnValue %[[#INSERT_RES_1]]
-
-declare { i8, i1 } @llvm.umul.with.overflow.i8(i8, i8)
-
-declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32)
-
+declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.umul.with.overflow.i32(i32, i32)
 declare {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll
new file mode 100644
index 0000000000000..963dd70f606b6
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/usub.with.overflow.ll
@@ -0,0 +1,89 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Void:.*]] = OpTypeVoid
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Function %[[Char]]
+; CHECK-DAG: %[[StructChar:.*]] = OpTypeStruct %[[Char]] %[[Char]]
+; CHECK-DAG: %[[ZeroChar:.*]] = OpConstant %[[Char]] 0
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[PtrInt:.*]] = OpTypePointer Function %[[Int]]
+; CHECK-DAG: %[[StructInt:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[ZeroInt:.*]] = OpConstant %[[Int]] 0
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[V2Bool:.*]] = OpTypeVector %[[Bool]] 2
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[V2Long:.*]] = OpTypeVector %[[Long]] 2
+; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]]
+; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]]
+; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[B:.*]] = OpFunctionParameter %[[Char]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[Struct:.*]] = OpISubBorrow %[[StructChar]] %[[A]] %[[B]]
+; CHECK: %[[Val:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Char]] %[[Struct]] 1
+; CHECK: %[[IsOver:.*]] = OpINotEqual %[[Bool]] %[[Over]] %[[ZeroChar]]
+; CHECK: %[[Res:.*]] = OpSelect %[[Char]] %[[IsOver]] %[[ZeroChar]] %[[Val]]
+; CHECK: OpStore %[[Ptr]] %[[Res]] Aligned 1
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i8(i8 zeroext %a, i8 zeroext %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i8, i1 } @llvm.usub.with.overflow.i8(i8 %a, i8 %b)
+  %cmp = extractvalue { i8, i1 } %umul, 1
+  %umul.value = extractvalue { i8, i1 } %umul, 0
+  %storemerge = select i1 %cmp, i8 0, i8 %umul.value
+  store i8 %storemerge, ptr %c, align 1
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[B2:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr2:.*]] = OpFunctionParameter %[[PtrInt]]
+; CHECK: %[[Struct2:.*]] = OpISubBorrow %[[StructInt]] %[[B2]] %[[A2]]
+; CHECK: %[[Val2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 0
+; CHECK: %[[Over2:.*]] = OpCompositeExtract %[[Int]] %[[Struct2]] 1
+; CHECK: %[[IsOver2:.*]] = OpINotEqual %[[Bool]] %[[Over2]] %[[ZeroInt]]
+; CHECK: %[[Res2:.*]] = OpSelect %[[Int]] %[[IsOver2]] %[[ZeroInt]] %[[Val2]]
+; CHECK: OpStore %[[Ptr2]] %[[Res2]] Aligned 4
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_i32(i32 %a, i32 %b, ptr nocapture %c) local_unnamed_addr {
+entry:
+  %umul = tail call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %b, i32 %a)
+  %umul.val = extractvalue { i32, i1 } %umul, 0
+  %umul.ov = extractvalue { i32, i1 } %umul, 1
+  %spec.select = select i1 %umul.ov, i32 0, i32 %umul.val
+  store i32 %spec.select, ptr %c, align 4
+  ret void
+}
+
+; CHECK: OpFunction
+; CHECK: %[[A3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[B3:.*]] = OpFunctionParameter %[[V2Long]]
+; CHECK: %[[Ptr3:.*]] = OpFunctionParameter %[[PtrV2Long]]
+; CHECK: %[[Struct3:.*]] = OpISubBorrow %[[StructV2Long]] %[[A3]] %[[B3]]
+; CHECK: %[[Val3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 0
+; CHECK: %[[Over3:.*]] = OpCompositeExtract %[[V2Long]] %[[Struct3]] 1
+; CHECK: %[[IsOver3:.*]] = OpINotEqual %[[V2Bool]] %[[Over3]] %[[ZeroV2Long]]
+; CHECK: %[[Res3:.*]] = OpSelect %[[V2Long]] %[[IsOver3]] %[[ZeroV2Long]] %[[Val3]]
+; CHECK: OpStore %[[Ptr3]] %[[Res3]] Aligned 16
+; CHECK: OpReturn
+define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) nounwind {
+  %umul = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> %a, <2 x i64> %b)
+  %umul.val = extractvalue {<2 x i64>, <2 x i1>} %umul, 0
+  %umul.ov = extractvalue {<2 x i64>, <2 x i1>} %umul, 1
+  %zero = alloca <2 x i64>, align 16
+  %spec.select = select <2 x i1> %umul.ov, <2 x i64> <i64 0, i64 0>, <2 x i64> %umul.val
+  store <2 x i64> %spec.select, ptr %p
+  ret void
+}
+
+declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8)
+declare {i32, i1} @llvm.usub.with.overflow.i32(i32, i32)
+declare {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
diff --git a/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll
new file mode 100644
index 0000000000000..1a630f77a44c5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/optimizations/add-check-overflow.ll
@@ -0,0 +1,56 @@
+; This test aims to check ability to support "Arithmetic with Overflow" intrinsics
+; in the special case when those intrinsics are being generated by the CodeGenPrepare;
+; pass during translations with optimization (note -O3 in llc arguments).
+
+; RUN: llc -O3 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; RUN: llc -O3 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O3 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[Val:.*]] "math"
+; CHECK-DAG: OpName %[[IsOver:.*]] "ov"
+; CHECK-DAG: %[[Int:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[PtrChar:.*]] = OpTypePointer Generic %[[Char]]
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[Struct:.*]] = OpTypeStruct %[[Int]] %[[Int]]
+; CHECK-DAG: %[[Const1:.*]] = OpConstant %[[Int]] 1
+; CHECK-DAG: %[[Const42:.*]] = OpConstant %[[Char]] 42
+; CHECK-DAG: %[[Zero:.*]] = OpConstantNull %[[Int]]
+
+; CHECK: OpFunction
+; CHECK: %[[A:.*]] = OpFunctionParameter %[[Int]]
+; CHECK: %[[Ptr:.*]] = OpFunctionParameter %[[PtrChar]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: OpBranch %[[#]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: %[[PhiRes:.*]] = OpPhi %[[Int]] %[[A]] %[[#]] %[[Val]] %[[#]]
+; CHECK: %[[AggRes:.*]] = OpIAddCarry %[[Struct]] %[[PhiRes]] %[[Const1]]
+; CHECK: %[[Val]] = OpCompositeExtract %[[Int]] %[[AggRes]] 0
+; CHECK: %[[Over:.*]] = OpCompositeExtract %[[Int]] %[[AggRes]] 1
+; CHECK: %[[IsOver]] = OpINotEqual %[[Bool:.*]] %[[Over]] %[[Zero]]
+; CHECK: OpBranchConditional %[[IsOver]] %[[#]] %[[#]]
+; CHECK: OpStore %[[Ptr]] %[[Const42]] Aligned 1
+; CHECK: OpBranch %[[#]]
+; CHECK: %[[#]] = OpLabel
+; CHECK: OpReturnValue %[[Val]]
+; CHECK: OpFunctionEnd
+
+define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) {
+entry:
+  br label %l1
+
+body:
+  store i8 42, ptr addrspace(4) %p
+  br label %l1
+
+l1:
+  %e = phi i32 [ %a, %entry ], [ %i, %body ]
+  %i = add nsw i32 %e, 1
+  %fl = icmp eq i32 %i, 0
+  br i1 %fl, label %exit, label %body
+
+exit:
+  ret i32 %i
+}
diff --git a/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll
new file mode 100644
index 0000000000000..cd4d9325c7659
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/passes/translate-aggregate-uaddo.ll
@@ -0,0 +1,64 @@
+; This test shows how value attributes are being passed during different translation steps.
+; See also test/CodeGen/SPIRV/optimizations/add-check-overflow.ll.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=prepare-functions 2>&1 | FileCheck %s  --check-prefix=CHECK-PREPARE
+; Intrinsics with aggregate return type are not substituted/removed.
+; CHECK-PREPARE: @llvm.uadd.with.overflow.i32
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=emit-intrinsics 2>&1 | FileCheck %s  --check-prefix=CHECK-IR
+; Aggregate data are wrapped into @llvm.fake.use(),
+; and their attributes are packed into a metadata for @llvm.spv.value.md().
+; CHECK-IR: %[[R1:.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32
+; CHECK-IR: call void @llvm.spv.value.md(metadata !0)
+; CHECK-IR: call void (...) @llvm.fake.use({ i32, i1 } %[[R1]])
+; CHECK-IR: %math = extractvalue { i32, i1 } %[[R1]], 0
+; CHECK-IR: %ov = extractvalue { i32, i1 } %[[R1]], 1
+; Type/Name attributes of the value.
+; CHECK-IR: !0 = !{{[{]}}!1, !""{{[}]}}
+; Origin data type of the value.
+; CHECK-IR: !1 = !{{[{]}}{{[{]}} i32, i1 {{[}]}} poison{{[}]}}
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=irtranslator 2>&1 | FileCheck %s  --check-prefix=CHECK-GMIR
+; Required info succeeded to get through IRTranslator.
+; CHECK-GMIR: %[[phires:.*]]:_(s32) = G_PHI
+; CHECK-GMIR: %[[math:.*]]:id(s32), %[[ov:.*]]:_(s1) = G_UADDO %[[phires]]:_, %[[#]]:_
+; CHECK-GMIR: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.spv.value.md), !0
+; CHECK-GMIR: FAKE_USE %[[math]]:id(s32), %[[ov]]:_(s1)
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=spirv-prelegalizer 2>&1 | FileCheck %s  --check-prefix=CHECK-PRE
+; Internal service instructions are consumed.
+; CHECK-PRE: G_UADDO
+; CHECK-PRE-NO: llvm.spv.value.md
+; CHECK-PRE-NO: FAKE_USE
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -print-after=instruction-select 2>&1 | FileCheck %s  --check-prefix=CHECK-ISEL
+; Names and types are restored and correctly encoded. Correct instruction selection is completed.
+; CHECK-ISEL-DAG: %[[int32:.*]]:type = OpTypeInt 32, 0
+; CHECK-ISEL-DAG: %[[struct:.*]]:type = OpTypeStruct %[[int32]]:type, %[[int32]]:type
+; CHECK-ISEL-DAG: %[[bool:.*]]:type = OpTypeBool
+; CHECK-ISEL-DAG: %[[zero32:.*]]:iid = OpConstantNull %[[int32]]:type
+; CHECK-ISEL-DAG: %[[res:.*]]:iid = OpIAddCarryS %[[struct]]:type
+; CHECK-ISEL-DAG: %[[math:.*]]:id = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 0
+; CHECK-ISEL-DAG: %[[ov32:.*]]:iid = OpCompositeExtract %[[int32]]:type, %[[res]]:iid, 1
+; CHECK-ISEL-DAG: %[[ov:.*]]:iid = OpINotEqual %[[bool]]:type, %[[ov32]]:iid, %[[zero32:.*]]:iid
+; CHECK-ISEL-DAG: OpName %[[math]]:id, 1752457581, 0
+; CHECK-ISEL-DAG: OpName %[[ov]]:iid, 30319
+
+define spir_func i32 @foo(i32 %a, ptr addrspace(4) %p) {
+entry:
+  br label %l1
+
+l1:                                               ; preds = %body, %entry
+  %e = phi i32 [ %a, %entry ], [ %math, %body ]
+  %0 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %e, i32 1)
+  %math = extractvalue { i32, i1 } %0, 0
+  %ov = extractvalue { i32, i1 } %0, 1
+  br i1 %ov, label %exit, label %body
+
+body:                                             ; preds = %l1
+  store i8 42, ptr addrspace(4) %p, align 1
+  br label %l1
+
+exit:                                             ; preds = %l1
+  ret i32 %math
+}
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels-empty-block.ll b/llvm/test/CodeGen/X86/basic-block-address-map-empty-block.ll
similarity index 83%
rename from llvm/test/CodeGen/X86/basic-block-sections-labels-empty-block.ll
rename to llvm/test/CodeGen/X86/basic-block-address-map-empty-block.ll
index 8e0f4fa7bc928..84948b7ecf6e0 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-labels-empty-block.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-empty-block.ll
@@ -1,5 +1,5 @@
 ;; This test verifies that with -gc-empty-basic-blocks SHT_LLVM_BB_ADDR_MAP will not include entries for empty blocks.
-; RUN: llc < %s -mtriple=x86_64 -O0 -basic-block-sections=labels -gc-empty-basic-blocks | FileCheck --check-prefix=CHECK %s
+; RUN: llc < %s -mtriple=x86_64 -O0 -basic-block-address-map -gc-empty-basic-blocks | FileCheck --check-prefix=CHECK %s
 
 define void @foo(i1 zeroext %0) nounwind {
   br i1 %0, label %2, label %empty_block
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels-empty-function.ll b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
similarity index 68%
rename from llvm/test/CodeGen/X86/basic-block-sections-labels-empty-function.ll
rename to llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
index 42d09212e6691..444655fc5299f 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-labels-empty-function.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll
@@ -1,6 +1,6 @@
 ;; Verify that the BB address map is not emitted for empty functions.
-; RUN: llc < %s -mtriple=x86_64 -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,BASIC
-; RUN: llc < %s -mtriple=x86_64 -basic-block-sections=labels -pgo-analysis-map=func-entry-count,bb-freq | FileCheck %s --check-prefixes=CHECK,PGO
+; RUN: llc < %s -mtriple=x86_64 -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC
+; RUN: llc < %s -mtriple=x86_64 -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq | FileCheck %s --check-prefixes=CHECK,PGO
 
 define void @empty_func() {
 entry:
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
index d7678604cffa2..9ff96381c2053 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll
@@ -1,4 +1,3 @@
-; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=labels | FileCheck %s
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-address-map | FileCheck %s
 
 $_Z4fooTIiET_v = comdat any
diff --git a/llvm/test/CodeGen/X86/basic-block-labels-mir-parse.mir b/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
similarity index 97%
rename from llvm/test/CodeGen/X86/basic-block-labels-mir-parse.mir
rename to llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
index 6408f0a30af7e..86f5f27494ec4 100644
--- a/llvm/test/CodeGen/X86/basic-block-labels-mir-parse.mir
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
@@ -1,5 +1,5 @@
 # Start after bbsections0-prepare and check that the BB address map is generated.
-# RUN: llc -mtriple x86_64-unknown-linux-gnu -start-after=bbsections-prepare  %s -o - | FileCheck %s -check-prefix=CHECK
+# RUN: llc -mtriple x86_64-unknown-linux-gnu -start-after=bbsections-prepare -basic-block-address-map %s -o - | FileCheck %s -check-prefix=CHECK
 
 # How to generate the input:
 # foo.cc
@@ -9,7 +9,7 @@
 # }
 #
 # clang -O0 -S -emit-llvm foo.cc
-# llc < foo.ll -stop-after=bbsections-prepare -basic-block-sections=labels
+# llc < foo.ll -stop-after=bbsections-prepare -basic-block-address-map
 
 # CHECK: .section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text
 --- |
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels-pgo-features.ll b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
similarity index 88%
rename from llvm/test/CodeGen/X86/basic-block-sections-labels-pgo-features.ll
rename to llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
index 92d3c88b4f601..73fe4f6ffedb0 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-labels-pgo-features.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll
@@ -1,13 +1,13 @@
 ; Check the basic block sections labels option
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,BASIC
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC
 
 ;; Also verify this holds for all PGO features enabled
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO-ALL,PGO-FEC,PGO-BBF,PGO-BRP
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO-ALL,PGO-FEC,PGO-BBF,PGO-BRP
 
 ;; Also verify that pgo extension only includes the enabled feature
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -pgo-analysis-map=func-entry-count | FileCheck %s --check-prefixes=CHECK,PGO-FEC,FEC-ONLY
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -pgo-analysis-map=bb-freq | FileCheck %s --check-prefixes=CHECK,PGO-BBF,BBF-ONLY
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -pgo-analysis-map=br-prob | FileCheck %s --check-prefixes=CHECK,PGO-BRP,BRP-ONLY
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=func-entry-count | FileCheck %s --check-prefixes=CHECK,PGO-FEC,FEC-ONLY
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=bb-freq | FileCheck %s --check-prefixes=CHECK,PGO-BBF,BBF-ONLY
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -pgo-analysis-map=br-prob | FileCheck %s --check-prefixes=CHECK,PGO-BRP,BRP-ONLY
 
 
 define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 !prof !0  {
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll
index 6ab24b494936a..4f12258eeeea0 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map.ll
+++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll
@@ -1,9 +1,7 @@
 ; Check the basic block sections labels option
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,UNIQ
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,UNIQ
 ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,NOUNIQ
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-sections=labels | FileCheck %s --check-prefixes=CHECK,NOUNIQ
-; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels -split-machine-functions | FileCheck %s --check-prefixes=CHECK,UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -split-machine-functions | FileCheck %s --check-prefixes=CHECK,UNIQ
 
 define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 {
   br i1 %0, label %3, label %8
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll b/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
index 1767903561ce1..fec87656be195 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-mir-print.ll
@@ -1,5 +1,5 @@
 ; Stop after bbsections-prepare and check MIR output for section type.
-; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=labels -stop-after=bbsections-prepare | FileCheck %s -check-prefix=BBLABELS
+; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-address-map -stop-after=bbsections-prepare | FileCheck %s -check-prefix=BBADDRMAP
 ; RUN: echo '!_Z3foob' > %t
 ; RUN: echo '!!1' >> %t
 ; RUN: echo '!!2' >> %t
@@ -33,7 +33,7 @@ define dso_local i32 @_Z3foob(i1 zeroext %0) {
 ; BBSECTIONS: bb.1 (%ir-block.7, bb_id 1)
 ; BBSECTIONS: bb.2 (%ir-block.8, bbsections 1, bb_id 2):
 
-; BBLABELS: bb.0 (%ir-block.1, bb_id 0):
-; BBLABELS: bb.1 (%ir-block.7, bb_id 1):
-; BBLABELS: bb.2 (%ir-block.8, bb_id 2):
-; BBLABELS: bb.3 (%ir-block.9, bb_id 3):
+; BBADDRMAP: bb.0 (%ir-block.1, bb_id 0):
+; BBADDRMAP: bb.1 (%ir-block.7, bb_id 1):
+; BBADDRMAP: bb.2 (%ir-block.8, bb_id 2):
+; BBADDRMAP: bb.3 (%ir-block.9, bb_id 3):
diff --git a/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll b/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
new file mode 100644
index 0000000000000..05d3f70820fb0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ipra-local-linkage-2.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=x86_64-- < %s | FileCheck --check-prefix=X64 %s
+; RUN: llc --mtriple=i386-- < %s | FileCheck --check-prefix=X86 %s
+
+; This test is to ensure rbp/rbx/ebp/esi is correctly saved/restored before clobbered when enable ipra.
+
+define internal void @callee_clobber_rbp() nounwind norecurse {
+; X64-LABEL: callee_clobber_rbp:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    #APP
+; X64-NEXT:    xorl %ebp, %ebp
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+;
+; X86-LABEL: callee_clobber_rbp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    #APP
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    #NO_APP
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  call void asm sideeffect "xor %ebp, %ebp", "~{ebp}"()
+  ret void
+}
+
+define internal void @callee_clobber_rbx(ptr %addr) nounwind norecurse {
+; X64-LABEL: callee_clobber_rbx:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    #APP
+; X64-NEXT:    xorl %ebx, %ebx
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    retq
+  call void asm sideeffect "xor %ebx, %ebx", "~{ebx}"()
+  ret void
+}
+
+define internal void @callee_clobber_esi(ptr %addr) nounwind norecurse {
+; X86-LABEL: callee_clobber_esi:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    #APP
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    #NO_APP
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  call void asm sideeffect "xor %esi, %esi", "~{esi}"()
+  ret void
+}
+
+define void @caller_use_rbp() "frame-pointer"="all" nounwind {
+; X64-LABEL: caller_use_rbp:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    movq %rsp, %rbp
+; X64-NEXT:    subq $16, %rsp
+; X64-NEXT:    callq callee_clobber_rbp
+; X64-NEXT:    movl $5, -4(%rbp)
+; X64-NEXT:    addq $16, %rsp
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+;
+; X86-LABEL: caller_use_rbp:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll callee_clobber_rbp
+; X86-NEXT:    movl $5, -4(%ebp)
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  call void @callee_clobber_rbp()
+  %addr = alloca i32, align 4
+  store i32 5, ptr %addr, align 4
+  ret void
+}
+
+define void @caller_use_rbx(i32 %X) nounwind ssp {
+; X64-LABEL: caller_use_rbx:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbp
+; X64-NEXT:    movq %rsp, %rbp
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    andq $-32, %rsp
+; X64-NEXT:    subq $64, %rsp
+; X64-NEXT:    movq %rsp, %rbx
+; X64-NEXT:    movq __stack_chk_guard(%rip), %rax
+; X64-NEXT:    movq %rax, 32(%rbx)
+; X64-NEXT:    movq %rsp, %rax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    leaq 15(,%rcx,4), %rcx
+; X64-NEXT:    andq $-16, %rcx
+; X64-NEXT:    subq %rcx, %rax
+; X64-NEXT:    movq %rax, %rsp
+; X64-NEXT:    movq %rbx, %rdi
+; X64-NEXT:    callq callee_clobber_rbx
+; X64-NEXT:    movq __stack_chk_guard(%rip), %rax
+; X64-NEXT:    cmpq 32(%rbx), %rax
+; X64-NEXT:    jne .LBB4_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    leaq -8(%rbp), %rsp
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    popq %rbp
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB4_2:
+; X64-NEXT:    callq __stack_chk_fail@PLT
+  %realign = alloca i32, align 32
+  %addr = alloca i32, i32 %X
+  call void @callee_clobber_rbx(ptr %realign)
+  ret void
+}
+
+define void @caller_use_esi(i32 %X) nounwind ssp {
+; X86-LABEL: caller_use_esi:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-32, %esp
+; X86-NEXT:    subl $32, %esp
+; X86-NEXT:    movl %esp, %esi
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl __stack_chk_guard, %ecx
+; X86-NEXT:    movl %ecx, 16(%esi)
+; X86-NEXT:    movl %esp, %ecx
+; X86-NEXT:    shll $2, %eax
+; X86-NEXT:    subl %eax, %ecx
+; X86-NEXT:    movl %ecx, %esp
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll callee_clobber_esi
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    movl __stack_chk_guard, %eax
+; X86-NEXT:    cmpl 16(%esi), %eax
+; X86-NEXT:    jne .LBB5_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    leal -4(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB5_2:
+; X86-NEXT:    calll __stack_chk_fail
+  %realign = alloca i32, align 32
+  %addr = alloca i32, i32 %X
+  call void @callee_clobber_esi(ptr %realign)
+  ret void
+}
diff --git a/llvm/test/DebugInfo/ARM/align-func-start.ll b/llvm/test/DebugInfo/ARM/align-func-start.ll
new file mode 100644
index 0000000000000..ebdf4b200dfff
--- /dev/null
+++ b/llvm/test/DebugInfo/ARM/align-func-start.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mtriple=arm-none-eabi < %s | FileCheck %s
+; RUN: llc -mtriple=arm-none-eabi < %s | llvm-mc --triple=arm-none-eabi -mcpu=cortex-m3
+
+; Check that, when an aligned loop is the first thing in a function, we do not
+; emit an invalid .loc directive, which is rejected by the assembly parser.
+
+; CHECK-NOT: .loc    0
+; CHECK: .loc    1 2 3 prologue_end
+; CHECK-NOT: .loc 0
+
+define dso_local void @foo() "target-cpu"="cortex-m3" !dbg !8 {
+entry:
+  br label %while.body, !dbg !11
+
+while.body:
+  br label %while.body, !dbg !11
+}
+
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 20.0.0git (git@github.com:llvm/llvm-project.git 1c984b86b389bbc71c8c2988d1d707e2f32878bd)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/work/scratch")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DILocation(line: 2, column: 3, scope: !8)
diff --git a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
index 1790f761585c3..a9c20d774822e 100644
--- a/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
+++ b/llvm/test/DebugInfo/MIR/X86/call-site-gnu-vs-dwarf5-attrs.mir
@@ -30,6 +30,21 @@
 # RUN:     -debug-entry-values -mtriple=x86_64-unknown-unknown \
 # RUN:     -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -check-prefixes=CHECK-DWARF5
 
+## === DWARF3, tune for gdb ===
+# RUN: llc -emit-call-site-info -dwarf-version 3 -debugger-tune=gdb -filetype=obj \
+# RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s  \
+# RUN:     | llvm-dwarfdump - | FileCheck %s -implicit-check-not=DW_AT_call
+
+## === DWARF3, tune for lldb ===
+# RUN: llc -dwarf-version 3 -debugger-tune=lldb -emit-call-site-info -filetype=obj \
+# RUN:     -mtriple=x86_64-unknown-unknown -start-after=machineverifier -o - %s   \
+# RUN:     | llvm-dwarfdump - | FileCheck %s -implicit-check-not=DW_AT_GNU_call
+
+## === DWARF3, tune for sce ===
+# RUN: llc -emit-call-site-info -dwarf-version 3 -filetype=obj -debugger-tune=sce \
+# RUN:     -debug-entry-values -mtriple=x86_64-unknown-unknown \
+# RUN:     -start-after=machineverifier -o - %s | llvm-dwarfdump - | FileCheck %s -implicit-check-not=DW_AT_call
+
 ## This is based on the following reproducer:
 ##
 ## extern void fn();
diff --git a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
index 4ae0b78f160c8..2d927b18d976d 100644
--- a/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
+++ b/llvm/test/DebugInfo/NVPTX/cu-range-hole.ll
@@ -120,6 +120,10 @@ entry:
 ; CHECK-NEXT: .b8 3                                // Abbreviation Code
 ; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
 ; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                               // DW_AT_address_class
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                // DW_AT_location
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
 ; CHECK-NEXT: .b8 3                                // DW_AT_name
 ; CHECK-NEXT: .b8 8                                // DW_FORM_string
 ; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
@@ -145,12 +149,12 @@ entry:
 ; CHECK-NEXT: }
 ; CHECK-NEXT: .section .debug_info
 ; CHECK-NEXT: {
-; CHECK-NEXT: .b32 183                             // Length of Unit
+; CHECK-NEXT: .b32 197                             // Length of Unit
 ; CHECK-NEXT: .b8 2                                // DWARF version number
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
 ; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
-; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0xb0 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0xbe DW_TAG_compile_unit
 ; CHECK-NEXT: .b8 99                               // DW_AT_producer
 ; CHECK-NEXT: .b8 108
 ; CHECK-NEXT: .b8 97
@@ -223,7 +227,7 @@ entry:
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
 ; CHECK-NEXT: .b64 $L__func_end2                   // DW_AT_high_pc
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x65:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x65:0x2e DW_TAG_subprogram
 ; CHECK-NEXT: .b64 $L__func_begin0                 // DW_AT_low_pc
 ; CHECK-NEXT: .b64 $L__func_end0                   // DW_AT_high_pc
 ; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
@@ -233,16 +237,23 @@ entry:
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
 ; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
-; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b32 193                             // DW_AT_type
 ; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x82:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x82:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 177
+; CHECK-NEXT: .b8 228
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
 ; CHECK-NEXT: .b8 99                               // DW_AT_name
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b32 193                             // DW_AT_type
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x8c:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x93:0x2e DW_TAG_subprogram
 ; CHECK-NEXT: .b64 $L__func_begin2                 // DW_AT_low_pc
 ; CHECK-NEXT: .b64 $L__func_end2                   // DW_AT_high_pc
 ; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
@@ -252,16 +263,23 @@ entry:
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
 ; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
-; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b32 193                             // DW_AT_type
 ; CHECK-NEXT: .b8 1                                // DW_AT_external
-; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xa9:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xb0:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 2                                // DW_AT_address_class
+; CHECK-NEXT: .b8 5                                // DW_AT_location
+; CHECK-NEXT: .b8 144
+; CHECK-NEXT: .b8 177
+; CHECK-NEXT: .b8 228
+; CHECK-NEXT: .b8 149
+; CHECK-NEXT: .b8 1
 ; CHECK-NEXT: .b8 101                              // DW_AT_name
 ; CHECK-NEXT: .b8 0
 ; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
 ; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
-; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b32 193                             // DW_AT_type
 ; CHECK-NEXT: .b8 0                                // End Of Children Mark
-; CHECK-NEXT: .b8 4                                // Abbrev [4] 0xb3:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0xc1:0x7 DW_TAG_base_type
 ; CHECK-NEXT: .b8 105                              // DW_AT_name
 ; CHECK-NEXT: .b8 110
 ; CHECK-NEXT: .b8 116
diff --git a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
index c25742ef0d276..3585095151181 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-addr-class.ll
@@ -160,6 +160,10 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 5                                   // Abbreviation Code
 ; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 2                                   // DW_AT_location
+; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
 ; CHECK-NEXT:.b8 3                                   // DW_AT_name
 ; CHECK-NEXT:.b8 8                                   // DW_FORM_string
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
@@ -171,6 +175,19 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
 ; CHECK-NEXT:.b8 6                                   // Abbreviation Code
+; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT:.b8 3                                   // DW_AT_name
+; CHECK-NEXT:.b8 8                                   // DW_FORM_string
+; CHECK-NEXT:.b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 73                                  // DW_AT_type
+; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT:.b8 0                                   // EOM(1)
+; CHECK-NEXT:.b8 0                                   // EOM(2)
+; CHECK-NEXT:.b8 7                                   // Abbreviation Code
 ; CHECK-NEXT:.b8 15                                  // DW_TAG_pointer_type
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
 ; CHECK-NEXT:.b8 73                                  // DW_AT_type
@@ -178,15 +195,15 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
 ; CHECK-NEXT:.b8 0                                   // EOM(2)
 ; CHECK-NEXT:.b8 0                                   // EOM(3)
-; CHECK-NEXT:        }
-; CHECK-NEXT:        .section        .debug_info
-; CHECK-NEXT:        {
-; CHECK-NEXT:.b32 240                                // Length of Unit
+; CHECK-NEXT:	}
+; CHECK-NEXT:	.section	.debug_info
+; CHECK-NEXT:	{
+; CHECK-NEXT:.b32 254                                // Length of Unit
 ; CHECK-NEXT:.b8 2                                   // DWARF version number
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b32 .debug_abbrev                      // Offset Into Abbrev. Section
 ; CHECK-NEXT:.b8 8                                   // Address Size (in bytes)
-; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0xe9 DW_TAG_compile_unit
+; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0xf7 DW_TAG_compile_unit
 ; CHECK-NEXT:.b8 99                                  // DW_AT_producer
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 97
@@ -298,7 +315,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 9                                   // DW_AT_location
 ; CHECK-NEXT:.b8 3
 ; CHECK-NEXT:.b64 SHARED
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0xa0:0x45 DW_TAG_subprogram
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0xa0:0x53 DW_TAG_subprogram
 ; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_frame_base
@@ -316,32 +333,46 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xc0:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xc0:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
+; CHECK-NEXT:.b8 5                                   // DW_AT_location
+; CHECK-NEXT:.b8 144
+; CHECK-NEXT:.b8 177
+; CHECK-NEXT:.b8 204
+; CHECK-NEXT:.b8 149
+; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 229                                // DW_AT_type
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xc9:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 243                                // DW_AT_type
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0xd0:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 120                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 238                                // DW_AT_type
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xd2:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 252                                // DW_AT_type
+; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0xd9:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 121                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 238                                // DW_AT_type
-; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xdb:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 252                                // DW_AT_type
+; CHECK-NEXT:.b8 5                                   // Abbrev [5] 0xe2:0x10 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
+; CHECK-NEXT:.b8 5                                   // DW_AT_location
+; CHECK-NEXT:.b8 144
+; CHECK-NEXT:.b8 177
+; CHECK-NEXT:.b8 228
+; CHECK-NEXT:.b8 149
+; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 105                                 // DW_AT_name
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b32 127                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xe5:0x9 DW_TAG_base_type
+; CHECK-NEXT:.b8 3                                   // Abbrev [3] 0xf3:0x9 DW_TAG_base_type
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 111
@@ -350,8 +381,8 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 6                                   // Abbrev [6] 0xee:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 229                                // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfc:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 243                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:        }
 ; CHECK-NEXT:         .section        .debug_loc      {       }
diff --git a/llvm/test/DebugInfo/NVPTX/debug-info.ll b/llvm/test/DebugInfo/NVPTX/debug-info.ll
index 9948925db57c9..643ed6484ae9f 100644
--- a/llvm/test/DebugInfo/NVPTX/debug-info.ll
+++ b/llvm/test/DebugInfo/NVPTX/debug-info.ll
@@ -490,6 +490,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 27                                  // Abbreviation Code
 ; CHECK-NEXT:.b8 5                                   // DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT:.b8 51                                  // DW_AT_address_class
+; CHECK-NEXT:.b8 11                                  // DW_FORM_data1
+; CHECK-NEXT:.b8 2                                   // DW_AT_location
+; CHECK-NEXT:.b8 10                                  // DW_FORM_block1
 ; CHECK-NEXT:.b8 49                                  // DW_AT_abstract_origin
 ; CHECK-NEXT:.b8 19                                  // DW_FORM_ref4
 ; CHECK-NEXT:.b8 0                                   // EOM(1)
@@ -703,12 +707,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:	}
 ; CHECK-NEXT:	.section	.debug_info
 ; CHECK-NEXT:	{
-; CHECK-NEXT:.b32 10029                              // Length of Unit
+; CHECK-NEXT:.b32 10036                              // Length of Unit
 ; CHECK-NEXT:.b8 2                                   // DWARF version number
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b32 .debug_abbrev                      // Offset Into Abbrev. Section
 ; CHECK-NEXT:.b8 8                                   // Address Size (in bytes)
-; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0x2726 DW_TAG_compile_unit
+; CHECK-NEXT:.b8 1                                   // Abbrev [1] 0xb:0x272d DW_TAG_compile_unit
 ; CHECK-NEXT:.b8 0                                   // DW_AT_producer
 ; CHECK-NEXT:.b8 4                                   // DW_AT_language
 ; CHECK-NEXT:.b8 0
@@ -2600,7 +2604,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
 ; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x84d:0x5 DW_TAG_pointer_type
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 23                                  // Abbrev [23] 0x852:0xbf DW_TAG_subprogram
+; CHECK-NEXT:.b8 23                                  // Abbrev [23] 0x852:0xc6 DW_TAG_subprogram
 ; CHECK-NEXT:.b64 $L__func_begin0                    // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__func_end0                      // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_frame_base
@@ -2634,7 +2638,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 22                                  // Abbrev [22] 0x886:0x9 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 0
@@ -2658,7 +2662,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 25                                  // Abbrev [25] 0x8aa:0x18 DW_TAG_inlined_subroutine
 ; CHECK-NEXT:.b32 707                                // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp0                           // DW_AT_low_pc
@@ -2680,867 +2684,874 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 37                                  // DW_AT_call_column
-; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8f2:0x1e DW_TAG_inlined_subroutine
+; CHECK-NEXT:.b8 26                                  // Abbrev [26] 0x8f2:0x25 DW_TAG_inlined_subroutine
 ; CHECK-NEXT:.b32 2066                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b64 $L__tmp9                           // DW_AT_low_pc
 ; CHECK-NEXT:.b64 $L__tmp10                          // DW_AT_high_pc
 ; CHECK-NEXT:.b8 1                                   // DW_AT_call_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_call_line
 ; CHECK-NEXT:.b8 5                                   // DW_AT_call_column
-; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x90a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 27                                  // Abbrev [27] 0x90a:0xc DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 2                                   // DW_AT_address_class
+; CHECK-NEXT:.b8 5                                   // DW_AT_location
+; CHECK-NEXT:.b8 144
+; CHECK-NEXT:.b8 179
+; CHECK-NEXT:.b8 204
+; CHECK-NEXT:.b8 149
+; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b32 2095                               // DW_AT_abstract_origin
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x911:0x588 DW_TAG_namespace
+; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x918:0x588 DW_TAG_namespace
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x916:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x91d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 202                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3737                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x91d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3744                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x924:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3781                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x924:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3788                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x92b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 204                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3810                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x92b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3817                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x932:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 205                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3841                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x932:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3848                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x939:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 206                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3870                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x939:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3877                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x940:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 207                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3901                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x940:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3908                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x947:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 208                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3930                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x947:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3937                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x94e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 209                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3967                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x94e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 3974                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x955:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3998                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x955:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4005                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x95c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 211                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4027                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x95c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4034                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x963:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 212                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4056                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x963:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4063                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x96a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 213                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4099                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x96a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4106                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x971:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4126                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x971:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4133                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x978:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 215                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4155                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x978:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4162                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x97f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 216                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4182                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x97f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4189                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x986:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 217                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4211                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x986:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4218                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x98d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 218                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4238                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x98d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4245                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x994:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 219                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4267                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x994:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4274                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x99b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 220                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4298                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x99b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4305                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 221                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4327                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4334                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 222                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4362                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9a9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4369                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b0:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 223                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4393                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b0:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4400                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b7:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 224                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4432                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9b7:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4439                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9be:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 225                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4467                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9be:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4474                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9c5:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 226                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4502                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9c5:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4509                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9cc:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 227                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4537                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9cc:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4544                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9d3:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 228                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4586                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9d3:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4593                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9da:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 229                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4629                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9da:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4636                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e1:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4666                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4673                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e8:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 231                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4697                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9e8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4704                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9ef:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 232                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4742                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9ef:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4749                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9f6:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 233                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4787                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9f6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4794                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9fd:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 234                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4843                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x9fd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4850                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa04:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 235                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4874                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa04:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4881                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa0b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 236                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4913                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa0b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4920                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa12:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 237                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4963                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa12:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 4970                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa19:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 238                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5017                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa19:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5024                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa20:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 239                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5048                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa20:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5055                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa27:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 240                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5085                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa27:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5092                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa2e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5135                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa2e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5142                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa35:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 242                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5176                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa35:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5183                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa3c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 243                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5213                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa3c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5220                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa43:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 244                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5246                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa43:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5253                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa4a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 245                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5277                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa4a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5284                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa51:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 246                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5310                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa51:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5317                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa58:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 247                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5337                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa58:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5344                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa5f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 248                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5368                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa5f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5375                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa66:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 249                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5399                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa66:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5406                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa6d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 250                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5428                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa6d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5435                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa74:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 251                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5457                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa74:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5464                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa7b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 252                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5488                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa7b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5495                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa82:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 253                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5521                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa82:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5528                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa89:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 254                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5556                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa89:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5563                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xa90:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 255                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5592                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa90:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5599                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa97:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 0                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5649                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa98:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5656                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xa9f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 1                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5680                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaa0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5687                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaa7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 2                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5719                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaa8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5726                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaaf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 3                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5764                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xab0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5771                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xab7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5797                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xab8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5804                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xabf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5842                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xac0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5849                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xac7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5888                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xac8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5895                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xacf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5917                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xad0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5924                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xad7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5948                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xad8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5955                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xadf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 9                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5989                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xae0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 5996                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xae7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 10                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6028                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xae8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6035                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaef:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 11                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6063                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaf0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6070                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaf7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6090                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaf8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6097                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xaff:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6119                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb00:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6126                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb07:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 14                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6148                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb08:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6155                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb0f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 15                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6175                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb10:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6182                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb17:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 16                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6204                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb18:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6211                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb1f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 17                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6237                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb20:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6244                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb27:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6268                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb27:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6275                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb2e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6288                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb2e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6295                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb35:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6308                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb35:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6315                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb3c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6328                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb3c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6335                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb43:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 180                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6354                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb43:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6361                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb4a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 199                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6374                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb4a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6381                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb51:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 218                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6393                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb51:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6400                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xb58:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 237                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6413                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb58:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6420                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb5f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 0                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6432                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb60:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6439                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb67:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 19                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6452                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb68:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6459                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb6f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 38                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6473                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb70:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6480                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb77:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 59                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6498                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb78:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6505                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb7f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6524                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb80:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6531                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb87:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 97                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6550                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb88:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6557                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb8f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6569                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb90:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6576                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb97:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 135                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6590                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb98:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6597                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xb9f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 147                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6620                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xba0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6627                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xba7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6644                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xba8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6651                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbaf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6663                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbb0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6670                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbb7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 222                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6683                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbb8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6690                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbbf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6703                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbc0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6710                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xbc7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 6                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 6722                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbc8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6729                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbcf:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6742                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbcf:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6749                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbd6:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 119                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6757                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbd6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6764                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbdd:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6805                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbdd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6812                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbe4:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 122                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6818                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbe4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6825                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbeb:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 123                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6838                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbeb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6845                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 129                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6867                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6874                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 130                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6887                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xbf9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6894                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc00:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 131                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6908                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc00:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6915                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc07:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 132                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6929                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc07:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 6936                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc0e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 133                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7057                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc0e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7064                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc15:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 134                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7085                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc15:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7092                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc1c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 135                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7110                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc1c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7117                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc23:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 136                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7128                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc23:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7135                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc2a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 137                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7145                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc2a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7152                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc31:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 138                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7173                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc31:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7180                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc38:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 139                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7194                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc38:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7201                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc3f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7220                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc3f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7227                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc46:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 142                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7243                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc46:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7250                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc4d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 143                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7270                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc4d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7277                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc54:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 144                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7321                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc54:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7328                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc5b:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 146                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7354                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc5b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7361                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc62:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 152                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7387                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc62:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7394                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc69:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7402                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc69:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7409                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc70:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 154                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7431                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc70:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7438                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc77:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7449                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc77:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7456                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc7e:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7481                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc7e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7488                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc85:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7513                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc85:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7520                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc8c:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 158                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7546                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc8c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7553                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc93:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 160                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7569                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc93:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7576                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc9a:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7614                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xc9a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7621                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca1:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 241                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7762                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7769                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca8:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 243                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7811                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xca8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7818                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcaf:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 245                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7830                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcaf:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7837                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcb6:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 246                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7716                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcb6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7723                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcbd:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 247                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7852                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcbd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7859                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcc4:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 249                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7879                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcc4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7886                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xccb:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 250                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7994                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xccb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8001                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd2:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 251                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7901                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7908                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd9:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 252                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7934                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xcd9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7941                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0xce0:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 253                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8021                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xce0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8028                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xce7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 149                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8064                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xce8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8071                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcef:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 150                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8096                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcf0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8103                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcf7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 151                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8130                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcf8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8137                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xcff:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 152                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8162                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd00:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8169                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd07:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8196                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd08:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8203                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd0f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 154                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8236                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd10:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8243                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd17:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8268                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd18:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8275                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd1f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8302                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd20:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8309                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd27:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8334                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd28:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8341                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd2f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 158                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8366                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd30:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8373                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd37:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8412                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd38:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8419                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd3f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 160                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8442                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd40:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8449                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd47:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8474                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd48:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8481                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd4f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 162                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8506                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd50:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8513                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd57:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8536                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd58:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8543                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd5f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 164                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8568                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd60:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8575                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd67:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8598                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd68:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8605                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd6f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 166                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8632                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd70:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8639                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd77:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8664                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd78:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8671                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd7f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 168                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8702                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd80:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8709                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd87:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 169                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8736                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd88:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8743                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd8f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 170                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8778                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd90:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8785                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd97:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 171                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8816                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd98:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8823                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xd9f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 172                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8854                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xda0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8861                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xda7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 173                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8892                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xda8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8899                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdaf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 174                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8933                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdb0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8940                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdb7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 8973                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdb8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8980                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdbf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 176                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9007                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdc0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9014                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdc7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 177                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9047                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdc8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9054                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdcf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 178                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9083                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdd0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9090                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdd7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 179                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9119                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdd8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9126                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xddf:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 180                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9157                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xde0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9164                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xde7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9191                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xde8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9198                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdef:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 182                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9225                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdf0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9232                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdf7:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9257                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdf8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9264                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xdff:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9289                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe00:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9296                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe07:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 185                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9319                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe08:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9326                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe0f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 186                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9353                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe10:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9360                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe17:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9389                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe18:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9396                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe1f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 188                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9428                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe20:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9435                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe27:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 189                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9471                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe28:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9478                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe2f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 190                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9520                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe30:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9527                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe37:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 191                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9556                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe38:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9563                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe3f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 192                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9605                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe40:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9612                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe47:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 193                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9654                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe48:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9661                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe4f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 194                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9686                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe50:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9693                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe57:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 195                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9720                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe58:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9727                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe5f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 196                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9764                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe60:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9771                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe67:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 197                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9806                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe68:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9813                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe6f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 198                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9836                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe70:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9843                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe77:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 199                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9868                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe78:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9875                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe7f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 200                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9900                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe80:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9907                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe87:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 201                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9930                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe88:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9937                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe8f:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 202                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9962                               // DW_AT_import
-; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe90:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 9969                               // DW_AT_import
+; CHECK-NEXT:.b8 30                                  // Abbrev [30] 0xe97:0x8 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 13                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 203                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 9998                               // DW_AT_import
+; CHECK-NEXT:.b32 10005                              // DW_AT_import
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xe99:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xea0:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3556,12 +3567,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 44                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xeae:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xeb5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0xeb4:0x11 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0xebb:0x11 DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -3578,7 +3589,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xec5:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xecc:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3598,10 +3609,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 46                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xedc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xee3:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xee2:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xee9:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3623,10 +3634,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 48                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xefb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf02:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf01:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf08:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3646,10 +3657,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 50                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf18:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf1f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf1e:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf25:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3671,10 +3682,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 52                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf37:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf3e:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf3d:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf44:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3694,10 +3705,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 56                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf54:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf5b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf5a:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf61:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3720,12 +3731,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 54                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf74:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf7b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf79:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf80:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf7f:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf86:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3747,10 +3758,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf98:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xf9f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xf9e:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfa5:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3770,10 +3781,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 60                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfb5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfbc:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfbb:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfc2:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3793,10 +3804,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 62                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfd2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfd9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfd8:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0xfdf:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3825,12 +3836,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 64                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xff8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xfff:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0xffd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1004:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1003:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x100a:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3848,10 +3859,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1018:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x101f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x101e:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1025:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3871,10 +3882,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 68                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1035:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x103c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x103b:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1042:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3892,10 +3903,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1050:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1057:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1056:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x105d:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3915,10 +3926,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 70                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x106d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1074:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1073:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x107a:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3936,10 +3947,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 76                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1088:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x108f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x108e:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1095:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3959,10 +3970,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10a5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10ac:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10ab:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10b2:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -3984,10 +3995,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 78                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10c4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10cb:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10ca:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10d1:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4007,10 +4018,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 80                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10e1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10e8:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10e7:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x10ee:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4031,12 +4042,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 82                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x10ff:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1106:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1104:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x110b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x110a:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1111:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4058,10 +4069,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 84                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1123:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x112a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1129:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1130:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4081,14 +4092,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 86                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1140:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1147:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1145:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x114c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x114a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1151:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1150:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1157:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4109,12 +4120,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 88                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1168:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x116f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x116d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1174:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1173:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x117a:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4135,12 +4146,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 90                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x118b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1192:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1190:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1197:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1196:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x119d:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4161,12 +4172,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 92                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11ae:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11b5:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11b3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11ba:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11b9:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11c0:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4197,19 +4208,19 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 94                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11dd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x11e4:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x11e3:0x7 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x11ea:0x7 DW_TAG_base_type
 ; CHECK-NEXT:.b8 105                                 // DW_AT_name
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11ea:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x11f1:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4233,14 +4244,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 96                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1205:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x120c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x120a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1211:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4631                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1210:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1215:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1217:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x121c:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4263,12 +4274,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 98                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x122f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1236:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1234:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x123b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x123a:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1241:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4288,12 +4299,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 100                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1253:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x125a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1259:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1260:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4319,12 +4330,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1278:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x127f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x127e:0x8 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1285:0x8 DW_TAG_base_type
 ; CHECK-NEXT:.b8 98                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 111
@@ -4332,7 +4343,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 2                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1286:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x128d:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4361,14 +4372,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12a8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12af:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12ad:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12b4:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12b3:0x38 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12ba:0x38 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4408,14 +4419,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e7:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12e5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x12ec:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12eb:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x12f2:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4435,12 +4446,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 108                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1304:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x130b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x130a:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1311:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4463,14 +4474,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 112                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1326:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x132d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x132b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1332:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1331:0x32 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1338:0x32 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4504,14 +4515,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 111                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1358:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x135f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x135d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1364:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1363:0x36 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x136a:0x36 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4549,14 +4560,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 114                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x138e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1395:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1393:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x139a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1399:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13a0:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4576,12 +4587,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13b2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13b9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13b8:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13bf:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4607,12 +4618,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13d7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x13de:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13dd:0x32 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x13e4:0x32 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4646,14 +4657,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 120                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1404:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x140b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1409:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1410:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x140f:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1416:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4671,12 +4682,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1426:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x142d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x142c:0xc DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1433:0xc DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -4688,7 +4699,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1438:0x25 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x143f:0x25 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4711,12 +4722,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 123                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1452:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1459:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1457:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x145e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x145d:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1464:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4740,10 +4751,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 125                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1478:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x147f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x147e:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1485:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4763,12 +4774,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 126                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1497:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x149e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x149d:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14a4:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4790,12 +4801,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 128                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14b8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14bf:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14be:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14c5:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4813,10 +4824,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 138                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14d3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14da:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14d9:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14e0:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4838,10 +4849,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 130                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14f2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x14f9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14f8:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x14ff:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4863,10 +4874,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 132                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1511:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1518:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1517:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x151e:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4886,10 +4897,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 134                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x152e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1535:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1534:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x153b:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4909,10 +4920,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 136                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x154b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1552:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1551:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1558:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4932,12 +4943,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 140                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x156a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1571:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1570:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1577:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4959,12 +4970,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 142                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x158b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1592:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1591:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1598:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -4988,12 +4999,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 143                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15ae:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15b5:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15b4:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15bb:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5015,12 +5026,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 145                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15cd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d4:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15d9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2125                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15d8:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x15df:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5038,12 +5049,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 146                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15ef:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x15f6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x15f5:0xa DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x15fc:0xa DW_TAG_base_type
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 117
@@ -5053,11 +5064,11 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x15ff:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5636                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1604:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 5641                               // DW_AT_type
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1609:0x8 DW_TAG_base_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1606:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5643                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x160b:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 5648                               // DW_AT_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1610:0x8 DW_TAG_base_type
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 104
 ; CHECK-NEXT:.b8 97
@@ -5065,7 +5076,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 8                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 1                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1611:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1618:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5087,10 +5098,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 147                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x162a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1631:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1630:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1637:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5120,10 +5131,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 149                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1651:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1658:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1657:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x165e:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5154,12 +5165,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 151                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1679:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1680:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x167e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1685:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1684:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x168b:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5178,12 +5189,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 155                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x169a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16a1:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x169f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16a5:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16ac:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5214,12 +5225,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 157                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16c7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16ce:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16cc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16d3:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16d2:0x2e DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x16d9:0x2e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5246,14 +5257,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 159                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f7:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16f5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16fc:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x16fa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1701:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4631                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1700:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1707:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5273,10 +5284,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 161                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1717:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x171e:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x171d:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1724:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5298,10 +5309,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 163                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1736:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x173d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x173c:0x29 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1743:0x29 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5328,12 +5339,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 165                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x175a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1761:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x175f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1766:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1765:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x176c:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5358,12 +5369,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 167                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1781:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1788:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1786:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x178d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x178c:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1793:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5387,12 +5398,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 169                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 4734                               // DW_AT_type
+; CHECK-NEXT:.b32 4741                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17a9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17b0:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17af:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17b6:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5410,10 +5421,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 171                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17c4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17cb:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17ca:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17d1:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5433,10 +5444,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 173                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17e1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17e8:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17e7:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x17ee:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5456,10 +5467,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x17fe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1805:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1804:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x180b:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5477,10 +5488,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 177                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1819:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1820:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x181f:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1826:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5500,10 +5511,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 179                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1836:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x183d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x183c:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1843:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5527,10 +5538,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1857:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x185e:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x185d:0x1f DW_TAG_subprogram
+; CHECK-NEXT:.b8 31                                  // Abbrev [31] 0x1864:0x1f DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -5552,10 +5563,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1876:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x187d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x187c:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1883:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 111
@@ -5563,13 +5574,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 54                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x188a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1891:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1890:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1897:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 105
@@ -5577,13 +5588,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 56                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x189e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18a5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18a4:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18ab:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 97
@@ -5591,13 +5602,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 58                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18b2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18b9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18b8:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18bf:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 97
@@ -5606,15 +5617,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 60                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18c7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18cc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18ce:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18d3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18d2:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18d9:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 105
@@ -5622,26 +5633,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 178                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18e0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18e7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18e6:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18ed:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 63                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18f3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x18fa:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x18f9:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1900:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 115
@@ -5649,26 +5660,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 72                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1907:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x190e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x190d:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1914:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 101                                 // DW_AT_name
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 112
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 100                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x191a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1921:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1920:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1927:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 98
@@ -5676,13 +5687,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 181                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x192e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1935:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1934:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x193b:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 111
@@ -5691,13 +5702,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 184                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1943:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x194a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1949:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1950:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 111
@@ -5705,15 +5716,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1957:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x195c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x195e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1963:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1962:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1969:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 101
@@ -5722,15 +5733,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 103                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1971:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1976:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1978:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x197d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4631                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x197c:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1983:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 101
@@ -5739,28 +5750,28 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x198b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1990:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1992:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1997:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1996:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x199d:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 103
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19a3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19aa:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19a9:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19b0:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 103
@@ -5769,13 +5780,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 112                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19b8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19bf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19be:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19c5:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 100
@@ -5783,45 +5794,45 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 115                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19cc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6615                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x19d7:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19dc:0x18 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19d8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6622                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x19de:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19e3:0x18 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 112                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 119
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 153                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19e9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19ee:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19f0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x19f5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19f4:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x19fb:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 65                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a01:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a08:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a07:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a0e:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 110
@@ -5829,13 +5840,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 74                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a15:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a1c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a1b:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a22:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 113
 ; CHECK-NEXT:.b8 114
@@ -5843,26 +5854,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 156                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a29:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a30:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a2f:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a36:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 116                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 67                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a3c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a43:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a42:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1a49:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 116                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
@@ -5870,14 +5881,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 76                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a50:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1a57:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a56:0xd DW_TAG_typedef
-; CHECK-NEXT:.b32 6755                               // DW_AT_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a5d:0xd DW_TAG_typedef
+; CHECK-NEXT:.b32 6762                               // DW_AT_type
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 118
@@ -5886,10 +5897,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 101                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1a63:0x2 DW_TAG_structure_type
+; CHECK-NEXT:.b8 34                                  // Abbrev [34] 0x1a6a:0x2 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a65:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 6771                               // DW_AT_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1a6c:0xe DW_TAG_typedef
+; CHECK-NEXT:.b32 6778                               // DW_AT_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 105
@@ -5899,35 +5910,35 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1a73:0x22 DW_TAG_structure_type
+; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1a7a:0x22 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 16                                  // DW_AT_byte_size
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 105                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a77:0xf DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a7e:0xf DW_TAG_member
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 117
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 107                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a86:0xe DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1a8d:0xe DW_TAG_member
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 108                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 36                                  // Abbrev [36] 0x1a95:0xd DW_TAG_subprogram
+; CHECK-NEXT:.b8 36                                  // Abbrev [36] 0x1a9c:0xd DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 111
@@ -5940,7 +5951,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1aa2:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1aa9:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 115
@@ -5948,13 +5959,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ab0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ab7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ab6:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1abd:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 101
@@ -5965,16 +5976,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ac7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6861                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ace:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6868                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1acd:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 6866                               // DW_AT_type
-; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1ad2:0x1 DW_TAG_subroutine_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1ad3:0x14 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1ad4:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 6873                               // DW_AT_type
+; CHECK-NEXT:.b8 38                                  // Abbrev [38] 0x1ad9:0x1 DW_TAG_subroutine_type
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1ada:0x14 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -5982,13 +5993,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 9                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 26                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ae1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ae8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ae7:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1aee:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -5997,13 +6008,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 22                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1af6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1afd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1afc:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1b03:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -6012,13 +6023,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 27                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b0b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b12:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1b11:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1b18:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 98                                  // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 101
@@ -6029,26 +6040,26 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 10                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 20                                  // DW_AT_decl_line
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
+; CHECK-NEXT:.b32 6979                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b22:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6973                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b27:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6973                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b2c:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b31:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b36:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7014                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 39                                  // Abbrev [39] 0x1b3c:0x1 DW_TAG_pointer_type
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b3d:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 6978                               // DW_AT_type
-; CHECK-NEXT:.b8 40                                  // Abbrev [40] 0x1b42:0x1 DW_TAG_const_type
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1b43:0xe DW_TAG_typedef
-; CHECK-NEXT:.b32 6993                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b29:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6980                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b2e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6980                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b33:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b38:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b3d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7021                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 39                                  // Abbrev [39] 0x1b43:0x1 DW_TAG_pointer_type
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b44:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 6985                               // DW_AT_type
+; CHECK-NEXT:.b8 40                                  // Abbrev [40] 0x1b49:0x1 DW_TAG_const_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1b4a:0xe DW_TAG_typedef
+; CHECK-NEXT:.b32 7000                               // DW_AT_type
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 122
@@ -6058,7 +6069,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 11                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 62                                  // DW_AT_decl_line
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1b51:0x15 DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1b58:0x15 DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6079,8 +6090,8 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x1b66:0x16 DW_TAG_typedef
-; CHECK-NEXT:.b32 7036                               // DW_AT_type
+; CHECK-NEXT:.b8 20                                  // Abbrev [20] 0x1b6d:0x16 DW_TAG_typedef
+; CHECK-NEXT:.b32 7043                               // DW_AT_type
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 99
@@ -6098,16 +6109,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b7c:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7041                               // DW_AT_type
-; CHECK-NEXT:.b8 41                                  // Abbrev [41] 0x1b81:0x10 DW_TAG_subroutine_type
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b86:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6973                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b8b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6973                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1b91:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1b83:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7048                               // DW_AT_type
+; CHECK-NEXT:.b8 41                                  // Abbrev [41] 0x1b88:0x10 DW_TAG_subroutine_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b8d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6980                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1b92:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6980                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1b98:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 99                                  // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 108
@@ -6118,15 +6129,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 212                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
+; CHECK-NEXT:.b32 6979                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ba2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ba7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ba9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bae:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1bad:0x19 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1bb4:0x19 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 100                                 // DW_AT_name
 ; CHECK-NEXT:.b8 105
 ; CHECK-NEXT:.b8 118
@@ -6134,15 +6145,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 21                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6742                               // DW_AT_type
+; CHECK-NEXT:.b32 6749                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bbb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bc7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1bc6:0x12 DW_TAG_subprogram
+; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1bcd:0x12 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 101                                 // DW_AT_name
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 105
@@ -6154,10 +6165,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bd2:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bd9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1bd8:0x11 DW_TAG_subprogram
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1bdf:0x11 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 102                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 101
@@ -6168,10 +6179,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1be3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bea:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6979                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1be9:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1bf0:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 103                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 116
@@ -6182,15 +6193,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 52                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 7168                               // DW_AT_type
+; CHECK-NEXT:.b32 7175                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1bfa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c01:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c00:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 5641                               // DW_AT_type
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c05:0x15 DW_TAG_subprogram
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c07:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 5648                               // DW_AT_type
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c0c:0x15 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 98
@@ -6199,13 +6210,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c14:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c1b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c1a:0x1a DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c21:0x1a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 100
 ; CHECK-NEXT:.b8 105
@@ -6214,15 +6225,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 23                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6757                               // DW_AT_type
+; CHECK-NEXT:.b32 6764                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c29:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c2e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c30:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c35:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c34:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c3b:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 108
@@ -6233,13 +6244,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 210                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
+; CHECK-NEXT:.b32 6979                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c45:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c4c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c4b:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c52:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 108
@@ -6249,15 +6260,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 95                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c5b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c60:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c62:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c67:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c66:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c6d:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 115
@@ -6270,19 +6281,19 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 106                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c79:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7305                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c7e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c83:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c89:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7310                               // DW_AT_type
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1c8e:0xb DW_TAG_base_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c80:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7312                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c85:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1c8a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1c90:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7317                               // DW_AT_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1c95:0xb DW_TAG_base_type
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 104
@@ -6293,7 +6304,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 5                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 4                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1c99:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ca0:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 109                                 // DW_AT_name
 ; CHECK-NEXT:.b8 98
 ; CHECK-NEXT:.b8 116
@@ -6304,17 +6315,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 98                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1caa:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7305                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1caf:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1cba:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7312                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cb6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cbb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1cc1:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 115
 ; CHECK-NEXT:.b8 111
@@ -6326,16 +6337,16 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cc6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ccb:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ccd:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd5:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7014                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 43                                  // Abbrev [43] 0x1cdb:0xf DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cd7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cdc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7021                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 43                                  // Abbrev [43] 0x1ce2:0xf DW_TAG_subprogram
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 97
 ; CHECK-NEXT:.b8 110
@@ -6344,10 +6355,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 118                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1cea:0x1d DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1cf1:0x1d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 97
@@ -6359,15 +6370,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 224                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
+; CHECK-NEXT:.b32 6979                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1cfc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6972                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d01:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d03:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d08:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1d07:0x12 DW_TAG_subprogram
+; CHECK-NEXT:.b8 18                                  // Abbrev [18] 0x1d0e:0x12 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 114
 ; CHECK-NEXT:.b8 97
@@ -6379,10 +6390,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d13:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d1a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 619                                // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d19:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d20:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6392,17 +6403,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 164                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5621                               // DW_AT_type
+; CHECK-NEXT:.b32 5628                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d29:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d2e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1d34:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7168                               // DW_AT_type
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d39:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d30:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d35:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1d3b:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7175                               // DW_AT_type
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d40:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6412,17 +6423,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 183                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d49:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d4e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d53:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d59:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d50:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d55:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d5a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1d60:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6433,17 +6444,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 187                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 6993                               // DW_AT_type
+; CHECK-NEXT:.b32 7000                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d6a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d6f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d74:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d7a:0x17 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d71:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d76:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d7b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d81:0x17 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 121
 ; CHECK-NEXT:.b8 115
@@ -6454,13 +6465,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 205                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d8b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1d92:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d91:0x23 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1d98:0x23 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 115
@@ -6473,21 +6484,21 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 109                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1da4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7168                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1da9:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7604                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dae:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 6979                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1db4:0x5 DW_TAG_pointer_type
-; CHECK-NEXT:.b32 7609                               // DW_AT_type
-; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1db9:0x5 DW_TAG_const_type
-; CHECK-NEXT:.b32 7310                               // DW_AT_type
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1dbe:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dab:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7175                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1db0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7611                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1db5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 6986                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 12                                  // Abbrev [12] 0x1dbb:0x5 DW_TAG_pointer_type
+; CHECK-NEXT:.b32 7616                               // DW_AT_type
+; CHECK-NEXT:.b8 13                                  // Abbrev [13] 0x1dc0:0x5 DW_TAG_const_type
+; CHECK-NEXT:.b32 7317                               // DW_AT_type
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1dc5:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 119                                 // DW_AT_name
 ; CHECK-NEXT:.b8 99
 ; CHECK-NEXT:.b8 116
@@ -6498,15 +6509,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 102                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dcf:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7168                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dd4:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7310                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1dd6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7175                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ddb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7317                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x1dda:0x78 DW_TAG_namespace
+; CHECK-NEXT:.b8 28                                  // Abbrev [28] 0x1de1:0x78 DW_TAG_namespace
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 95
 ; CHECK-NEXT:.b8 103
@@ -6517,43 +6528,43 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 120
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1de5:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dec:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 201                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7762                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dec:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7769                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1df3:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 207                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7811                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1df3:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7818                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dfa:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 211                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7830                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1dfa:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7837                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e01:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 217                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7852                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e01:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7859                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e08:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 228                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7879                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e08:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7886                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e0f:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 229                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7901                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e0f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7908                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e16:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 230                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7934                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e16:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 7941                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e1d:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 232                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7994                               // DW_AT_import
-; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e1d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT:.b32 8001                               // DW_AT_import
+; CHECK-NEXT:.b8 29                                  // Abbrev [29] 0x1e24:0x7 DW_TAG_imported_declaration
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 233                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8021                               // DW_AT_import
-; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x1e24:0x2d DW_TAG_subprogram
+; CHECK-NEXT:.b32 8028                               // DW_AT_import
+; CHECK-NEXT:.b8 4                                   // Abbrev [4] 0x1e2b:0x2d DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 78
@@ -6581,17 +6592,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 8                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7762                               // DW_AT_type
+; CHECK-NEXT:.b32 7769                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e46:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e4b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e4d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e52:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1e52:0xf DW_TAG_typedef
-; CHECK-NEXT:.b32 7777                               // DW_AT_type
+; CHECK-NEXT:.b8 33                                  // Abbrev [33] 0x1e59:0xf DW_TAG_typedef
+; CHECK-NEXT:.b32 7784                               // DW_AT_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 100
@@ -6602,35 +6613,35 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 121                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1e61:0x22 DW_TAG_structure_type
+; CHECK-NEXT:.b8 35                                  // Abbrev [35] 0x1e68:0x22 DW_TAG_structure_type
 ; CHECK-NEXT:.b8 16                                  // DW_AT_byte_size
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 117                                 // DW_AT_decl_line
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e65:0xf DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e6c:0xf DW_TAG_member
 ; CHECK-NEXT:.b8 113                                 // DW_AT_name
 ; CHECK-NEXT:.b8 117
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 119                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e74:0xe DW_TAG_member
+; CHECK-NEXT:.b8 11                                  // Abbrev [11] 0x1e7b:0xe DW_TAG_member
 ; CHECK-NEXT:.b8 114                                 // DW_AT_name
 ; CHECK-NEXT:.b8 101
 ; CHECK-NEXT:.b8 109
 ; CHECK-NEXT:.b8 0
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 120                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 2                                   // DW_AT_data_member_location
 ; CHECK-NEXT:.b8 35
 ; CHECK-NEXT:.b8 8
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1e83:0x13 DW_TAG_subprogram
+; CHECK-NEXT:.b8 42                                  // Abbrev [42] 0x1e8a:0x13 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_name
 ; CHECK-NEXT:.b8 69
 ; CHECK-NEXT:.b8 120
@@ -6643,10 +6654,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
 ; CHECK-NEXT:.b8 1                                   // DW_AT_noreturn
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e90:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1e97:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1e96:0x16 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1e9d:0x16 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 97
@@ -6656,13 +6667,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ea6:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ead:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1eac:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1eb3:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 108
 ; CHECK-NEXT:.b8 100
@@ -6672,15 +6683,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 29                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 3
-; CHECK-NEXT:.b32 7762                               // DW_AT_type
+; CHECK-NEXT:.b32 7769                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ebc:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec1:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ec8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ec7:0x16 DW_TAG_subprogram
+; CHECK-NEXT:.b8 37                                  // Abbrev [37] 0x1ece:0x16 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 97                                  // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 111
@@ -6690,13 +6701,13 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 36                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 1
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ed7:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ede:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1edd:0x21 DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1ee4:0x21 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6707,17 +6718,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 209                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1eee:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef3:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef8:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1efe:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ef5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1efa:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1eff:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f05:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6729,17 +6740,17 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 214                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 7968                               // DW_AT_type
+; CHECK-NEXT:.b32 7975                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f10:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f15:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f1a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
-; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f20:0x1a DW_TAG_base_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f17:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f1c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f21:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
+; CHECK-NEXT:.b8 0                                   // End Of Children Mark
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f27:0x1a DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6765,7 +6776,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f3a:0x1b DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f41:0x1b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6778,12 +6789,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f4a:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f4f:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f51:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f56:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f55:0x1c DW_TAG_subprogram
+; CHECK-NEXT:.b8 32                                  // Abbrev [32] 0x1f5c:0x1c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 115                                 // DW_AT_name
 ; CHECK-NEXT:.b8 116
 ; CHECK-NEXT:.b8 114
@@ -6794,15 +6805,15 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 7                                   // DW_AT_decl_file
 ; CHECK-NEXT:.b8 175                                 // DW_AT_decl_line
-; CHECK-NEXT:.b32 8049                               // DW_AT_type
+; CHECK-NEXT:.b32 8056                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
 ; CHECK-NEXT:.b8 1                                   // DW_AT_external
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f66:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5631                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f6b:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 7476                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f6d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5638                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f72:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 7483                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f71:0xf DW_TAG_base_type
+; CHECK-NEXT:.b8 10                                  // Abbrev [10] 0x1f78:0xf DW_TAG_base_type
 ; CHECK-NEXT:.b8 108                                 // DW_AT_name
 ; CHECK-NEXT:.b8 111
 ; CHECK-NEXT:.b8 110
@@ -6817,7 +6828,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 0
 ; CHECK-NEXT:.b8 4                                   // DW_AT_encoding
 ; CHECK-NEXT:.b8 8                                   // DW_AT_byte_size
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1f80:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1f87:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6840,10 +6851,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1f9a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fa1:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fa0:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fa7:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6868,10 +6879,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fbc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fc3:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fc2:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fc9:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6894,10 +6905,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fdc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1fe3:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fe2:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x1fe9:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6922,10 +6933,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x1ffe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2005:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2004:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x200b:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6951,12 +6962,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2021:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2028:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2026:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x202d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x202c:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2033:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -6979,10 +6990,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2046:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x204d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x204c:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2053:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7007,10 +7018,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2068:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x206f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x206e:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2075:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7033,10 +7044,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2088:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x208f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x208e:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2095:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7059,10 +7070,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20a8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20af:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20ae:0x2e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20b5:0x2e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7094,12 +7105,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d8:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20d6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20dd:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20dc:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20e3:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7120,10 +7131,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20f4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x20fb:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x20fa:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2101:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7146,10 +7157,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2114:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x211b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x211a:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2121:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7172,10 +7183,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2134:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x213b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x213a:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2141:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7196,10 +7207,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2152:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2159:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2158:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x215f:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7222,10 +7233,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2172:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2179:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2178:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x217f:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7246,10 +7257,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2190:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2197:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2196:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x219d:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7274,10 +7285,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21b2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21b9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21b8:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21bf:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7300,10 +7311,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21d2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21d9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21d8:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21df:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7327,12 +7338,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21f3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21fa:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21f8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x21ff:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x21fe:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2205:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7357,10 +7368,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x221a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2221:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2220:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2227:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7383,14 +7394,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x223a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2241:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x223f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2246:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2244:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x224b:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x224a:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2251:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7414,12 +7425,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2265:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x226c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x226a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2271:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2270:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2277:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7443,12 +7454,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x228b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2292:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2290:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2297:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2296:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x229d:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7472,12 +7483,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b8:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22b6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22bd:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22bc:0x29 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22c3:0x29 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7504,12 +7515,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22da:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22e1:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22df:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x22e6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4631                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22e5:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x22ec:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7535,12 +7546,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2302:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2309:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2307:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x230e:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x230d:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2314:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7563,12 +7574,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 85                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2329:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2330:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x232f:0x28 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2336:0x28 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7594,12 +7605,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x234c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2353:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2351:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2358:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2357:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x235e:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7626,10 +7637,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2375:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x237c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x237b:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2382:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7654,12 +7665,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 125                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2399:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23a0:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x239f:0x26 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23a6:0x26 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7686,12 +7697,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 66                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 3764                               // DW_AT_type
+; CHECK-NEXT:.b32 3771                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23bf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23c6:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23c5:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23cc:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7716,10 +7727,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23e1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x23e8:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23e7:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x23ee:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7744,10 +7755,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2403:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x240a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2409:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2410:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7770,10 +7781,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2423:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x242a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2429:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2430:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7796,10 +7807,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2443:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x244a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2449:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2450:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7820,10 +7831,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2461:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2468:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2467:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x246e:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7846,12 +7857,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 116                                 // DW_AT_decl_line
 ; CHECK-NEXT:.b8 4
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2483:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x248a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2489:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2490:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7876,12 +7887,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 12                                  // DW_AT_decl_file
 ; CHECK-NEXT:.b8 71                                  // DW_AT_decl_line
 ; CHECK-NEXT:.b8 6
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24a7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24ae:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24ad:0x27 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24b4:0x27 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7906,12 +7917,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24c9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24d0:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24ce:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24d5:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2125                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24d4:0x2b DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24db:0x2b DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7945,10 +7956,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x24f9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2500:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x24ff:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2506:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -7983,12 +7994,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2525:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x252c:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x252a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2531:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2530:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2537:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8010,12 +8021,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2549:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2550:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x254e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2555:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2554:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x255b:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8050,12 +8061,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x257a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2581:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x257f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2586:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2585:0x31 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x258c:0x31 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8085,14 +8096,14 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25ad:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25ab:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b2:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b0:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4624                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25b7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4631                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25b6:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25bd:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8115,10 +8126,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25d0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25d7:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25d6:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25dd:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8143,10 +8154,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25f2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x25f9:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25f8:0x2c DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x25ff:0x2c DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8176,12 +8187,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2619:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2620:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x261e:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 5164                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2625:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 5171                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2624:0x2a DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x262b:0x2a DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8209,12 +8220,12 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2643:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x264a:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2648:0x5 DW_TAG_formal_parameter
-; CHECK-NEXT:.b32 4579                               // DW_AT_type
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x264f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b32 4586                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x264e:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2655:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8235,10 +8246,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2666:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x266d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x266c:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2673:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8261,10 +8272,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2686:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x268d:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x268c:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2693:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8287,10 +8298,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 3
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26ad:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26ac:0x1e DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26b3:0x1e DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8311,10 +8322,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 4
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26c4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26cb:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26ca:0x20 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26d1:0x20 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8337,10 +8348,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 5
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26e4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x26eb:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26ea:0x24 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x26f1:0x24 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8367,10 +8378,10 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 6
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2708:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x270f:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
-; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x270e:0x22 DW_TAG_subprogram
+; CHECK-NEXT:.b8 44                                  // Abbrev [44] 0x2715:0x22 DW_TAG_subprogram
 ; CHECK-NEXT:.b8 95                                  // DW_AT_MIPS_linkage_name
 ; CHECK-NEXT:.b8 90
 ; CHECK-NEXT:.b8 76
@@ -8395,7 +8406,7 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-NEXT:.b8 2
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 1                                   // DW_AT_declaration
-; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x272a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT:.b8 7                                   // Abbrev [7] 0x2731:0x5 DW_TAG_formal_parameter
 ; CHECK-NEXT:.b32 2116                               // DW_AT_type
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
 ; CHECK-NEXT:.b8 0                                   // End Of Children Mark
diff --git a/llvm/test/ExecutionEngine/MCJIT/test-global-ctors.ll b/llvm/test/ExecutionEngine/MCJIT/test-global-ctors.ll
index 2e95e7eda588a..4598c9f71f14c 100644
--- a/llvm/test/ExecutionEngine/MCJIT/test-global-ctors.ll
+++ b/llvm/test/ExecutionEngine/MCJIT/test-global-ctors.ll
@@ -1,6 +1,6 @@
 ; RUN: %lli -jit-kind=mcjit %s > /dev/null
 ; RUN: %lli %s > /dev/null
-; UNSUPPORTED: target={{.*}}-darwin{{.*}}
+; UNSUPPORTED: system-darwin
 @var = global i32 1, align 4
 @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @ctor_func, ptr null }]
 @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @dtor_func, ptr null }]
diff --git a/llvm/test/ExecutionEngine/Orc/weak-comdat.ll b/llvm/test/ExecutionEngine/Orc/weak-comdat.ll
index e1e1ab02e2981..4ebbfdf4aa35b 100644
--- a/llvm/test/ExecutionEngine/Orc/weak-comdat.ll
+++ b/llvm/test/ExecutionEngine/Orc/weak-comdat.ll
@@ -1,5 +1,5 @@
 ; RUN: lli -extra-module %p/Inputs/weak-comdat-def.ll %s
-; UNSUPPORTED: target={{.*}}-darwin{{.*}}
+; UNSUPPORTED: system-darwin
 
 declare i32 @g()
 
diff --git a/llvm/test/MC/AMDGPU/ds_swizzle.s b/llvm/test/MC/AMDGPU/ds_swizzle.s
new file mode 100644
index 0000000000000..d768339fe5b77
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/ds_swizzle.s
@@ -0,0 +1,131 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=bonaire -show-encoding %s | FileCheck -check-prefix=GFX7 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=tonga -show-encoding %s | FileCheck -check-prefix=GFX8 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx908 -show-encoding %s | FileCheck -check-prefix=GFX9 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10PLUS %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX10PLUS %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX10PLUS %s
+
+// RUN: not llvm-mc -triple=amdgcn -mcpu=bonaire -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR-PREGFX9 %s --implicit-check-not=error:
+// RUN: not llvm-mc -triple=amdgcn -mcpu=tonga -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR-PREGFX9 %s --implicit-check-not=error:
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx908 -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR %s --implicit-check-not=error:
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR %s --implicit-check-not=error:
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR %s --implicit-check-not=error:
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck -check-prefix=ERROR %s --implicit-check-not=error:
+
+//==============================================================================
+// FFT mode
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,5)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(FFT,5) ; encoding: [0x05,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,5) ; encoding: [0x05,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0x1f,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0x1f,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:0xf000
+// GFX7:   ds_swizzle_b32 v5, v1 offset:61440 ; encoding: [0x00,0xf0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+// GFX8:   ds_swizzle_b32 v5, v1 offset:61440 ; encoding: [0x00,0xf0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xf0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xf0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,32)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: FFT swizzle must be in the interval [0,31]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,-2)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: FFT swizzle must be in the interval [0,31]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a comma
+
+ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: FFT mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parentheses
+
+//==============================================================================
+// ROTATE mode
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// GFX9:      ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_swizzle_b32 v5, v1 offset:0xd000
+// GFX7:   ds_swizzle_b32 v5, v1 offset:53248 ; encoding: [0x00,0xd0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+// GFX8:   ds_swizzle_b32 v5, v1 offset:53248 ; encoding: [0x00,0xd0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX9:   ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xd0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+// GFX10PLUS: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xd0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,2,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: direction must be 0 (left) or 1 (right)
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,-1,31)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: direction must be 0 (left) or 1 (right)
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,32)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: number of threads to rotate must be in the interval [0,31]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,-2)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: number of threads to rotate must be in the interval [0,31]
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a comma
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a comma
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a comma
+
+ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1,2)
+// ERROR-PREGFX9: :[[@LINE-1]]:{{[0-9]+}}: error: Rotate mode swizzle not supported on this GPU
+// ERROR: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parentheses
diff --git a/llvm/test/MC/AMDGPU/gfx90a_ldst_acc.s b/llvm/test/MC/AMDGPU/gfx90a_ldst_acc.s
index ee8218613a6dc..0b518acc884df 100644
--- a/llvm/test/MC/AMDGPU/gfx90a_ldst_acc.s
+++ b/llvm/test/MC/AMDGPU/gfx90a_ldst_acc.s
@@ -8257,15 +8257,15 @@ ds_read_u16 a5, v1
 // NOT-GFX90A: :[[@LINE+1]]:{{[0-9]+}}: error: invalid register class: agpr loads and stores not supported on this GPU
 ds_read_u16 a5, v1 offset:4
 
-// GFX90A: ds_swizzle_b32 a5, v1 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0x05]
+// GFX90A: ds_swizzle_b32 a5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0x05]
 // NOT-GFX90A: :[[@LINE+1]]:{{[0-9]+}}: error: invalid register class: agpr loads and stores not supported on this GPU
 ds_swizzle_b32 a5, v1 offset:65535
 
-// GFX90A: ds_swizzle_b32 a255, v1 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0xff]
+// GFX90A: ds_swizzle_b32 a255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0xff]
 // NOT-GFX90A: :[[@LINE+1]]:{{[0-9]+}}: error: invalid register class: agpr loads and stores not supported on this GPU
 ds_swizzle_b32 a255, v1 offset:65535
 
-// GFX90A: ds_swizzle_b32 a5, v255 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0xff,0x00,0x00,0x05]
+// GFX90A: ds_swizzle_b32 a5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0xff,0x00,0x00,0x05]
 // NOT-GFX90A: :[[@LINE+1]]:{{[0-9]+}}: error: invalid register class: agpr loads and stores not supported on this GPU
 ds_swizzle_b32 a5, v255 offset:65535
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_ds.txt
index 688b5f916630a..bf9bad5240416 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx10_ds.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_ds.txt
@@ -4208,24 +4208,48 @@
 # GFX10: ds_sub_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00]
 0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00
 
-# GFX10: ds_swizzle_b32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
+# GFX10: ds_swizzle_b32 v255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff
 
 # GFX10: ds_swizzle_b32 v5, v1                   ; encoding: [0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX10: ds_swizzle_b32 v5, v1 offset:65535      ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX10: ds_swizzle_b32 v5, v1 offset:65535 gds  ; encoding: [0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05]
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) gds ; encoding: [0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05
 
 # GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(BITMASK_PERM,"00p00") ; encoding: [0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX10: ds_swizzle_b32 v5, v255 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
+# GFX10: ds_swizzle_b32 v5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05
 
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX10: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05
+
 # GFX10: ds_wrap_rtn_b32 v255, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xd0,0xd8,0x01,0x02,0x03,0xff]
 0xff,0xff,0xd0,0xd8,0x01,0x02,0x03,0xff
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_ds.txt
index 1b632b56400fb..448cfc95de095 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_ds.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_ds.txt
@@ -3802,16 +3802,16 @@
 # GFX11: ds_sub_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00]
 0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00
 
-# GFX11: ds_swizzle_b32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
+# GFX11: ds_swizzle_b32 v255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff
 
 # GFX11: ds_swizzle_b32 v5, v1                   ; encoding: [0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX11: ds_swizzle_b32 v5, v1 offset:65535      ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX11: ds_swizzle_b32 v5, v1 offset:65535 gds  ; encoding: [0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05]
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) gds ; encoding: [0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0xd6,0xd8,0x01,0x00,0x00,0x05
 
 # GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(BITMASK_PERM,"00p00") ; encoding: [0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
@@ -3829,9 +3829,33 @@
 # GFX11: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,32,1) ; encoding: [0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
 0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
 
-# GFX11: ds_swizzle_b32 v5, v255 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
+# GFX11: ds_swizzle_b32 v5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05
 
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX11: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05
+
 # GFX11: ds_wrap_rtn_b32 v255, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xd0,0xd8,0x01,0x02,0x03,0xff]
 0xff,0xff,0xd0,0xd8,0x01,0x02,0x03,0xff
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt
index 473ede00603a7..080a4cab2a319 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt
@@ -2415,13 +2415,13 @@
 # GFX12: ds_sub_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00]
 0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00
 
-# GFX12: ds_swizzle_b32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
+# GFX12: ds_swizzle_b32 v255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff
 
 # GFX12: ds_swizzle_b32 v5, v1                   ; encoding: [0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
 
-# GFX12: ds_swizzle_b32 v5, v1 offset:65535      ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05
 
 # GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(BITMASK_PERM,"00p00") ; encoding: [0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
@@ -2439,9 +2439,33 @@
 # GFX12: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,32,1) ; encoding: [0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
 0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
 
-# GFX12: ds_swizzle_b32 v5, v255 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
+# GFX12: ds_swizzle_b32 v5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
 0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05
 
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xe0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc0,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc4,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc3,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc7,0xd4,0xd8,0x01,0x00,0x00,0x05
+
 # GFX12: ds_store_2addr_b32 v0, v1, v2           ; encoding: [0x00,0x00,0x38,0xd8,0x00,0x01,0x02,0x00]
 0x00,0x00,0x38,0xd8,0x00,0x01,0x02,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_ldst_acc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_ldst_acc.txt
index b8cc7ac604da2..081b40ac78f8c 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx90a_ldst_acc.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx90a_ldst_acc.txt
@@ -6192,13 +6192,13 @@
 # GFX90A: ds_read_u16 a5, v1 offset:4     ; encoding: [0x04,0x00,0x78,0xda,0x01,0x00,0x00,0x05]
 0x04,0x00,0x78,0xda,0x01,0x00,0x00,0x05
 
-# GFX90A: ds_swizzle_b32 a5, v1 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0x05]
+# GFX90A: ds_swizzle_b32 a5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0x05]
 0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0x05
 
-# GFX90A: ds_swizzle_b32 a255, v1 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0xff]
+# GFX90A: ds_swizzle_b32 a255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0xff]
 0xff,0xff,0x7a,0xda,0x01,0x00,0x00,0xff
 
-# GFX90A: ds_swizzle_b32 a5, v255 offset:65535 ; encoding: [0xff,0xff,0x7a,0xda,0xff,0x00,0x00,0x05]
+# GFX90A: ds_swizzle_b32 a5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xda,0xff,0x00,0x00,0x05]
 0xff,0xff,0x7a,0xda,0xff,0x00,0x00,0x05
 
 # GFX90A: ds_swizzle_b32 a5, v1           ; encoding: [0x00,0x00,0x7a,0xda,0x01,0x00,0x00,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx9_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx9_ds.txt
index 58a44e1250542..2af4b8396d71a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx9_ds.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx9_ds.txt
@@ -1074,13 +1074,13 @@
 # CHECK: ds_read_u16 v5, v1 offset:65535 gds     ; encoding: [0xff,0xff,0x79,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0x79,0xd8,0x01,0x00,0x00,0x05
 
-# CHECK: ds_swizzle_b32 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0x05]
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0x05
 
-# CHECK: ds_swizzle_b32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0xff]
+# CHECK: ds_swizzle_b32 v255, v1 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0xff]
 0xff,0xff,0x7a,0xd8,0x01,0x00,0x00,0xff
 
-# CHECK: ds_swizzle_b32 v5, v255 offset:65535    ; encoding: [0xff,0xff,0x7a,0xd8,0xff,0x00,0x00,0x05]
+# CHECK: ds_swizzle_b32 v5, v255 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0x7a,0xd8,0xff,0x00,0x00,0x05]
 0xff,0xff,0x7a,0xd8,0xff,0x00,0x00,0x05
 
 # CHECK: ds_swizzle_b32 v5, v1                   ; encoding: [0x00,0x00,0x7a,0xd8,0x01,0x00,0x00,0x05]
@@ -1089,9 +1089,33 @@
 # CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(BITMASK_PERM,"00p00") ; encoding: [0x04,0x00,0x7a,0xd8,0x01,0x00,0x00,0x05]
 0x04,0x00,0x7a,0xd8,0x01,0x00,0x00,0x05
 
-# CHECK: ds_swizzle_b32 v5, v1 offset:65535 gds  ; encoding: [0xff,0xff,0x7b,0xd8,0x01,0x00,0x00,0x05]
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,31) gds ; encoding: [0xff,0xff,0x7b,0xd8,0x01,0x00,0x00,0x05]
 0xff,0xff,0x7b,0xd8,0x01,0x00,0x00,0x05
 
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,16) ; encoding: [0x10,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(FFT,0) ; encoding: [0x00,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xe0,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,0) ; encoding: [0x00,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,0) ; encoding: [0x00,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,1) ; encoding: [0x20,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc0,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,1) ; encoding: [0x20,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0x20,0xc4,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,0,31) ; encoding: [0xe0,0xc3,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc3,0x7a,0xd8,0x01,0x00,0x00,0x05
+
+# CHECK: ds_swizzle_b32 v5, v1 offset:swizzle(ROTATE,1,31) ; encoding: [0xe0,0xc7,0x7a,0xd8,0x01,0x00,0x00,0x05]
+0xe0,0xc7,0x7a,0xd8,0x01,0x00,0x00,0x05
+
 # CHECK: ds_permute_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x05]
 0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x05
 
diff --git a/llvm/test/MC/WebAssembly/type-checker-errors.s b/llvm/test/MC/WebAssembly/type-checker-errors.s
index 3106fe76c8449..5fdc2f56daf57 100644
--- a/llvm/test/MC/WebAssembly/type-checker-errors.s
+++ b/llvm/test/MC/WebAssembly/type-checker-errors.s
@@ -19,7 +19,7 @@ local_set_no_local_type:
 local_set_empty_stack_while_popping:
   .functype local_set_empty_stack_while_popping () -> ()
   .local i32
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   local.set 0
   end_function
 
@@ -27,7 +27,7 @@ local_set_type_mismatch:
   .functype local_set_type_mismatch () -> ()
   .local i32
   f32.const 1.0
-# CHECK: [[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   local.set 0
   end_function
 
@@ -40,7 +40,7 @@ local_tee_no_local_type:
 local_tee_empty_stack_while_popping:
   .functype local_tee_empty_stack_while_popping () -> ()
   .local f32
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got []
   local.tee 0
   end_function
 
@@ -48,8 +48,9 @@ local_tee_type_mismatch:
   .functype local_tee_type_mismatch () -> ()
   .local f32
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   local.tee 0
+  drop
   end_function
 
 global_get_missing_globaltype:
@@ -79,7 +80,7 @@ global_set_expected_expression_operand:
 global_set_empty_stack_while_popping:
   .functype global_set_empty_stack_while_popping () -> ()
   .globaltype valid_global, i64
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i64
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i64] but got []
   global.set valid_global
   end_function
 
@@ -87,7 +88,7 @@ global_set_type_mismatch:
   .functype global_set_type_mismatch () -> ()
   .globaltype valid_global, i64
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected i64
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i64] but got [i32]
   global.set valid_global
   end_function
 
@@ -109,46 +110,52 @@ table_get_missing_tabletype:
 
 table_get_empty_stack_while_popping:
   .functype table_get_empty_stack_while_popping () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.get valid_table
+  drop
   end_function
 
 table_get_type_mismatch:
   .functype table_get_type_mismatch () -> ()
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   table.get valid_table
+  drop
   end_function
 
 table_set_expected_expression_operand:
   .functype table_set_expected_expression_operand () -> ()
+  i32.const 0
 # CHECK: :[[@LINE+1]]:13: error: expected expression operand
   table.set 1
   end_function
 
 table_set_missing_tabletype:
   .functype table_set_missing_tabletype () -> ()
+  i32.const 0
 # CHECK: :[[@LINE+1]]:13: error: symbol foo: missing .tabletype
   table.set foo
   end_function
 
 table_set_empty_stack_while_popping_1:
   .functype table_set_empty_stack_while_popping_1 () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping externref
+# CHECK: :[[@LINE+2]]:3: error: type mismatch, expected [externref] but got []
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.set valid_table
   end_function
 
 table_set_empty_stack_while_popping_2:
   .functype table_set_empty_stack_while_popping_2 (externref) -> ()
   local.get 0
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.set valid_table
   end_function
 
 table_set_type_mismatch_1:
   .functype table_set_type_mismatch_1 () -> ()
+  i32.const 0
   ref.null_func
-# CHECK: :[[@LINE+1]]:3: error: popped funcref, expected externref
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [externref] but got [funcref]
   table.set valid_table
   end_function
 
@@ -156,32 +163,41 @@ table_set_type_mismatch_2:
   .functype table_set_type_mismatch_2 () -> ()
   f32.const 1.0
   ref.null_extern
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   table.set valid_table
   end_function
 
 table_fill_expected_expression_operand:
   .functype table_fill_expected_expression_operand () -> ()
+  i32.const 0
+  ref.null_extern
+  i32.const 4
 # CHECK: :[[@LINE+1]]:14: error: expected expression operand
   table.fill 1
   end_function
 
 table_fill_missing_tabletype:
   .functype table_fill_missing_tabletype () -> ()
+  i32.const 0
+  ref.null_extern
+  i32.const 4
 # CHECK: :[[@LINE+1]]:14: error: symbol foo: missing .tabletype
   table.fill foo
   end_function
 
 table_fill_empty_stack_while_popping_1:
   .functype table_fill_empty_stack_while_popping_1 () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+3]]:3: error: type mismatch, expected [i32] but got []
+# CHECK: :[[@LINE+2]]:3: error: type mismatch, expected [externref] but got []
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.fill valid_table
   end_function
 
 table_fill_empty_stack_while_popping_2:
   .functype table_fill_empty_stack_while_popping_2 (i32) -> ()
   local.get 0
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping externref
+# CHECK: :[[@LINE+2]]:3: error: type mismatch, expected [externref] but got []
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.fill valid_table
   end_function
 
@@ -189,22 +205,25 @@ table_fill_empty_stack_while_popping_3:
   .functype table_fill_empty_stack_while_popping_3 (i32, externref) -> ()
   local.get 1
   local.get 0
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.fill valid_table
   end_function
 
 table_fill_type_mismatch_1:
   .functype table_fill_type_mismatch_1 () -> ()
+  i32.const 0
+  ref.null_extern
   ref.null_func
-# CHECK: :[[@LINE+1]]:3: error: popped funcref, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [funcref]
   table.fill valid_table
   end_function
 
 table_fill_type_mismatch_2:
   .functype table_fill_type_mismatch_2 () -> ()
+  i32.const 0
   ref.null_func
   i32.const 1
-# CHECK: [[@LINE+1]]:3: error: popped funcref, expected externref
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [externref] but got [funcref]
   table.fill valid_table
   end_function
 
@@ -213,23 +232,16 @@ table_fill_type_mismatch_3:
   f32.const 2.0
   ref.null_extern
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   table.fill valid_table
   end_function
 
 table_fill_type_mismatch_4:
   .functype table_fill_type_mismatch_4 () -> ()
-  ref.null_exn
   i32.const 1
-# CHECK: [[@LINE+1]]:3: error: popped exnref, expected externref
-  table.fill valid_table
-  end_function
-
-table_fill_type_mismatch_5:
-  .functype table_fill_type_mismatch_5 () -> ()
   ref.null_exn
   i32.const 1
-# CHECK: [[@LINE+1]]:3: error: popped exnref, expected externref
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [externref] but got [exnref]
   table.fill valid_table
   end_function
 
@@ -244,14 +256,15 @@ table_grow_non_exist_table:
 table_grow_type_mismatch_1:
   .functype table_grow_type_mismatch_1 (externref, i32) -> (i32)
   local.get 1
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping externref
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [externref] but got []
   table.grow valid_table
   end_function
 
 table_grow_type_mismatch_2:
   .functype table_grow_type_mismatch_2 (externref, i32) -> (i32)
   local.get 0
-# CHECK: [[@LINE+1]]:3: error: popped externref, expected i32
+  local.get 0
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32] but got [externref]
   table.grow valid_table
   end_function
 
@@ -260,57 +273,62 @@ table_grow_wrong_result:
   local.get 0
   local.get 1
   table.grow valid_table
-# CHECK: [[@LINE+1]]:3: error: popped i32, expected f32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_function
 
 drop_empty_stack_while_popping:
   .functype drop_empty_stack_while_popping () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
 end_block_insufficient_values_on_stack_1:
   .functype end_block_insufficient_values_on_stack_1 () -> ()
   block i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_block
+  drop
   end_function
 
 end_block_insufficient_values_on_stack_2:
   .functype end_block_insufficient_values_on_stack_2 () -> ()
   block () -> (i32)
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_block
+  drop
   end_function
 
 end_block_type_mismatch:
   .functype end_block_type_mismatch () -> ()
   block i32
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: end got f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_block
+  drop
   end_function
 
 end_loop_insufficient_values_on_stack:
   .functype end_loop_insufficient_values_on_stack () -> ()
   loop i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_loop
+  drop
   end_function
 
 end_loop_type_mismatch:
   .functype end_loop_type_mismatch () -> ()
   loop f32
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: end got i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_loop
+  drop
   end_function
 
 end_if_insufficient_values_on_stack_1:
   .functype end_if_insufficient_values_on_stack_1 () -> ()
   i32.const 1
   if i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_if
   end_function
 
@@ -319,8 +337,9 @@ end_if_type_mismatch_1:
   i32.const 1
   if f32
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: end got i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_if
+  drop
   end_function
 
 end_if_insufficient_values_on_stack_2:
@@ -329,7 +348,7 @@ end_if_insufficient_values_on_stack_2:
   if i32
   i32.const 2
   else
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_if
   drop
   end_function
@@ -341,7 +360,7 @@ end_if_type_mismatch_2:
   i32.const 2
   else
   f32.const 3.0
-# CHECK: :[[@LINE+1]]:3: error: end got f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_if
   drop
   end_function
@@ -350,7 +369,7 @@ else_insufficient_values_on_stack:
   .functype else_insufficient_values_on_stack () -> ()
   i32.const 1
   if i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   else
   i32.const 0
   end_if
@@ -362,7 +381,7 @@ else_type_mismatch:
   i32.const 1
   if i32
   f32.const 0.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   else
   i32.const 0
   end_if
@@ -377,7 +396,7 @@ end_try_insufficient_values_on_stack:
   try i32
   i32.const 0
   catch_all
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_try
   drop
   end_function
@@ -387,7 +406,7 @@ end_try_type_mismatch:
   try i32
   i32.const 0
   catch tag_f32
-# CHECK: :[[@LINE+1]]:3: error: end got f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   end_try
   drop
   end_function
@@ -395,7 +414,7 @@ end_try_type_mismatch:
 catch_insufficient_values_on_stack:
   .functype catch_insufficient_values_on_stack () -> ()
   try i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   catch tag_i32
   end_try
   drop
@@ -405,7 +424,7 @@ catch_type_mismatch:
   .functype catch_type_mismatch () -> ()
   try i32
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   catch tag_i32
   end_try
   drop
@@ -414,7 +433,7 @@ catch_type_mismatch:
 catch_all_insufficient_values_on_stack:
   .functype catch_all_insufficient_values_on_stack () -> ()
   try i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   catch_all
   i32.const 0
   end_try
@@ -425,7 +444,7 @@ catch_all_type_mismatch:
   .functype catch_all_type_mismatch () -> ()
   try i32
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   catch_all
   i32.const 0
   end_try
@@ -435,7 +454,7 @@ catch_all_type_mismatch:
 delegate_insufficient_values_on_stack:
   .functype delegate_insufficient_values_on_stack () -> ()
   try i32
-# CHECK: :[[@LINE+1]]:3: error: end: insufficient values on the type stack
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   delegate 0
   drop
   end_function
@@ -444,46 +463,46 @@ delegate_type_mismatch:
   .functype delegate_type_mismatch () -> ()
   try i32
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: end got f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   delegate 0
   drop
   end_function
 
 end_function_empty_stack_while_popping:
   .functype end_function_empty_stack_while_popping () -> (i32)
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   end_function
 
 end_function_type_mismatch:
   .functype end_function_type_mismatch () -> (f32)
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   end_function
 
 end_function_superfluous_end_function_values:
   .functype end_function_superfluous_end_function_values () -> ()
   i32.const 1
   f32.const 2.0
-# CHECK: :[[@LINE+1]]:3: error: 2 superfluous return values
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [i32, f32]
   end_function
 
 return_empty_stack_while_popping:
   .functype return_empty_stack_while_popping () -> (i32)
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   return
   end_function
 
 return_type_mismatch:
   .functype return_type_mismatch () -> (f32)
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   return
   end_function
 
 # Missing index for call_indirect.
 call_indirect_empty_stack_while_popping_1:
   .functype call_indirect_empty_stack_while_popping_1 () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   call_indirect () -> ()
   end_function
 
@@ -491,7 +510,7 @@ call_indirect_empty_stack_while_popping_1:
 call_indirect_empty_stack_while_popping_2:
   .functype call_indirect_empty_stack_while_popping_1 (f32) -> ()
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got []
   call_indirect (f32) -> ()
   end_function
 
@@ -499,7 +518,7 @@ call_indirect_type_mismatch_for_argument:
   .functype call_indirect_type_mismatch_for_argument () -> ()
   i32.const 1
   i32.const 2
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got [i32]
   call_indirect (f32) -> ()
   end_function
 
@@ -507,13 +526,13 @@ call_indirect_superfluous_value_at_end:
   .functype call_indirect_superfluous_value_at_end () -> ()
   i32.const 1
   call_indirect () -> (i64)
-# CHECK: :[[@LINE+1]]:3: error: 1 superfluous return values
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [i64]
   end_function
 
 # Missing index for return_call_indirect.
 return_call_indirect_empty_stack_while_popping_1:
   .functype return_call_indirect_empty_stack_while_popping_1 () -> ()
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   return_call_indirect () -> ()
   end_function
 
@@ -521,7 +540,7 @@ return_call_indirect_empty_stack_while_popping_1:
 return_call_indirect_empty_stack_while_popping_2:
   .functype return_call_indirect_empty_stack_while_popping_2 () -> ()
   i32.const 1
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping f32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [f32] but got []
   return_call_indirect (f32) -> ()
   end_function
 
@@ -535,14 +554,14 @@ call_expected_expression_operand:
 
 call_empty_stack_while_popping:
   .functype call_empty_stack_while_popping () -> ()
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   call fn_i32_to_void
   end_function
 
 call_type_mismatch:
   .functype call_type_mismatch () -> ()
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   call fn_i32_to_void
   end_function
 
@@ -551,7 +570,7 @@ call_type_mismatch:
 call_superfluous_value_at_end:
   .functype call_superfluous_value_at_end () -> ()
   call fn_void_to_i32
-# CHECK: :[[@LINE+1]]:3: error: 1 superfluous return values
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [i32]
   end_function
 
 call_missing_functype:
@@ -568,14 +587,14 @@ return_call_expected_expression_operand:
 
 return_call_empty_stack_while_popping:
   .functype return_call_empty_stack_while_popping () -> ()
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   return_call fn_i32_to_void
   end_function
 
 return_call_type_mismatch:
   .functype return_call_type_mismatch () -> ()
   f32.const 1.0
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [f32]
   return_call fn_i32_to_void
   end_function
 
@@ -607,27 +626,29 @@ catch_superfluous_value_at_end:
   catch tag_i32
   end_try
 # FIXME: Superfluous value should be caught at end_try?
-# CHECK: :[[@LINE+1]]:3: error: 1 superfluous return values
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [i32]
   end_function
 
 ref_is_null_empty_stack_while_popping:
   .functype ref_is_null_empty_stack_while_popping () -> ()
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping reftype
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [ref] but got []
   ref.is_null
+  drop
   end_function
 
 ref_is_null_type_mismatch:
   .functype ref_is_null_type_mismatch () -> ()
   i32.const 1
-# CHECK: [[@LINE+1]]:3: error: popped i32, expected reftype
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [ref] but got [i32]
   ref.is_null
+  drop
   end_function
 
 ref_is_null_pushes_i32:
   .functype ref_is_null_pushes_i32 () -> (i64)
   ref.null_func
   ref.is_null
-# CHECK: :[[@LINE+1]]:3: error: popped i32, expected i64
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i64] but got [i32]
   end_function
 
 # For the other instructions, the type checker checks vs the operands in the
@@ -636,16 +657,18 @@ ref_is_null_pushes_i32:
 
 other_insn_test_1:
   .functype other_insn_test_1 () -> ()
-# CHECK: [[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: [[@LINE+1]]:3: error: type mismatch, expected [i32, i32] but got []
   i32.add
+  drop
   end_function
 
 other_insn_test_2:
   .functype other_insn_test_2 () -> ()
   i32.const 1
   ref.null_func
-# CHECK: :[[@LINE+1]]:3: error: popped funcref, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32, i32] but got [i32, funcref]
   i32.add
+  drop
   end_function
 
 other_insn_test_3:
@@ -653,7 +676,7 @@ other_insn_test_3:
   f32.const 1.0
   f32.const 2.0
   f32.add
-# CHECK: :[[@LINE+1]]:3: error: 1 superfluous return values
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [] but got [f32]
   end_function
 
 # Unreachable code within 'block' does not affect type checking after
@@ -663,7 +686,7 @@ check_after_unreachable_within_block:
   block
   unreachable
   end_block
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
@@ -673,7 +696,7 @@ check_after_unreachable_within_loop:
   loop
   unreachable
   end_loop
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
@@ -686,7 +709,7 @@ check_after_unreachable_within_if_1:
   else
   unreachable
   end_if
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
@@ -697,7 +720,7 @@ check_after_unreachable_within_if_2:
   if
   unreachable
   else
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_if
   end_function
@@ -710,7 +733,7 @@ check_after_unreachable_within_try_1:
   catch_all
   unreachable
   end_try
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
@@ -721,7 +744,7 @@ check_after_unreachable_within_try_2:
   unreachable
   catch tag_i32
   drop
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_try
   end_function
@@ -732,7 +755,7 @@ check_after_unreachable_within_try_3:
   try
   unreachable
   catch_all
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_try
   end_function
@@ -743,7 +766,7 @@ check_after_unreachable_within_try_4:
   try
   unreachable
   delegate 0
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping value
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [any] but got []
   drop
   end_function
 
@@ -753,7 +776,7 @@ br_invalid_type_loop:
   loop (i32) -> (f32)
     drop
     f32.const 1.0
-# CHECK: :[[@LINE+1]]:5: error: br got f32, expected i32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [i32] but got [f32]
     br 0
   end_loop
   drop
@@ -763,7 +786,7 @@ br_invalid_type_block:
   .functype br_invalid_type_block () -> ()
   i32.const 1
   block (i32) -> (f32)
-# CHECK: :[[@LINE+1]]:5: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [f32] but got [i32]
     br 0
     f32.const 1.0
   end_block
@@ -777,7 +800,7 @@ br_invalid_type_if:
     f32.const 1.0
   else
     i32.const 1
-# CHECK: :[[@LINE+1]]:5: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [f32] but got [i32]
     br 0
   end_if
   drop
@@ -787,7 +810,7 @@ br_invalid_type_try:
   .functype br_invalid_type_try () -> ()
   try f32
     i32.const 1
-# CHECK: :[[@LINE+1]]:5: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [f32] but got [i32]
     br 0
   catch tag_f32
   end_try
@@ -799,7 +822,7 @@ br_invalid_type_catch:
   try f32
     f32.const 1.0
   catch tag_i32
-# CHECK: :[[@LINE+1]]:5: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [f32] but got [i32]
     br 0
   end_try
   drop
@@ -811,7 +834,7 @@ br_invalid_type_catch_all:
     f32.const 1.0
   catch_all
     i32.const 1
-# CHECK: :[[@LINE+1]]:5: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [f32] but got [i32]
     br 0
   end_try
   drop
@@ -834,7 +857,7 @@ br_incorrect_signature:
   block f32
     block i32
       i32.const 1
-# CHECK: :[[@LINE+1]]:7: error: br got i32, expected f32
+# CHECK: :[[@LINE+1]]:7: error: type mismatch, expected [f32] but got [i32]
       br 1
     end_block
     drop
@@ -847,7 +870,7 @@ br_incorrect_func_signature:
   .functype br_incorrect_func_signature () -> (i32)
   block f32
     f32.const 1.0
-# CHECK: :[[@LINE+1]]:5: error: br got f32, expected i32
+# CHECK: :[[@LINE+1]]:5: error: type mismatch, expected [i32] but got [f32]
     br 1
   end_block
   drop
@@ -856,20 +879,29 @@ br_incorrect_func_signature:
 
 multiple_errors_in_function:
   .functype multiple_errors_in_function () -> ()
-# CHECK: :[[@LINE+2]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+2]]:3: error: type mismatch, expected [i32] but got []
 # CHECK: :[[@LINE+1]]:13: error: expected expression operand
   table.get 1
 
-# CHECK: :[[@LINE+3]]:3: error: empty stack while popping i32
-# CHECK: :[[@LINE+2]]:3: error: empty stack while popping externref
-# CHECK: :[[@LINE+1]]:3: error: empty stack while popping i32
+# CHECK: :[[@LINE+3]]:3: error: type mismatch, expected [i32] but got []
+# CHECK: :[[@LINE+2]]:3: error: type mismatch, expected [externref] but got []
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got []
   table.fill valid_table
 
   f32.const 0.0
   ref.null_extern
-# CHECK: :[[@LINE+2]]:3: error: popped externref, expected i32
-# CHECK: :[[@LINE+1]]:3: error: popped f32, expected i32
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32, i32] but got [f32, externref]
   i32.add
   drop
+  end_function
+
+.functype take_and_return_multi(i32, i64, f32, f64) -> (i32, i64, f32, f64)
 
+call_with_multi_param_and_return:
+  .functype call_with_multi_param_and_return () -> (i32)
+  ref.null_extern
+  f32.const 0.0
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32, i64, f32, f64] but got [externref, f32]
+  call take_and_return_multi
+# CHECK: :[[@LINE+1]]:3: error: type mismatch, expected [i32] but got [i32, i64, f32, f64]
   end_function
diff --git a/llvm/test/MC/WebAssembly/type-checker-return.s b/llvm/test/MC/WebAssembly/type-checker-return.s
index 552093bc555bd..016c034058101 100644
--- a/llvm/test/MC/WebAssembly/type-checker-return.s
+++ b/llvm/test/MC/WebAssembly/type-checker-return.s
@@ -1,10 +1,5 @@
 # RUN: llvm-mc -triple=wasm32 -mattr=+tail-call %s 2>&1
 
-# XFAIL: *
-
-# FIXME: These shouldn't produce an error, as return will implicitly drop any
-# superfluous values.
-
 return_superfluous_return_values:
   .functype return_superfluous_return_values () -> ()
   i32.const 1
diff --git a/llvm/test/Support/interrupts.test b/llvm/test/Support/interrupts.test
index 4768ac61dff02..0966586106cc7 100644
--- a/llvm/test/Support/interrupts.test
+++ b/llvm/test/Support/interrupts.test
@@ -1,9 +1,9 @@
 ## Show that SIGINT and similar signals don't cause crash messages to be
 ## reported.
 # RUN: %python %s wrapper llvm-symbolizer 2> %t.err
-# RUN: FileCheck --input-file=%t.err %s
+# RUN: FileCheck -DMSG=%errc_ENOENT --input-file=%t.err %s
 
-# CHECK: {{.*}} error: 'foo': {{[Nn]}}o such file or directory
+# CHECK: {{.*}} error: 'foo': [[MSG]]
 # CHECK-NOT: {{.+}}
 
 import os
diff --git a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
index 63366ba998c7b..ca757a165fa4b 100644
--- a/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/actual-arguments.ll
@@ -68,18 +68,14 @@ define internal i32 @test_cannot_promote_3(ptr %p, ptr nocapture readonly %test_
   ret i32 %sum
 }
 
-; FIXME: We should perform ArgPromotion here!
-;
 ; This is called only by @caller_safe_args_1, from which we can prove that
 ; %test_c does not alias %p for any Call to the function, so we can promote it.
 ;
 define internal i32 @test_can_promote_1(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_can_promote_1
-; CHECK-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[TEST_C:%.*]]) {
-; CHECK-NEXT:    [[TEST_C_VAL:%.*]] = load i32, ptr [[TEST_C]], align 4
-; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_VAL]])
-; CHECK-NEXT:    [[LTEST_C:%.*]] = load i32, ptr [[TEST_C]], align 4
-; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[LTEST_C]], [[RES]]
+; CHECK-SAME: (ptr [[P:%.*]], i32 [[TEST_C_0_VAL:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_0_VAL]])
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[TEST_C_0_VAL]], [[RES]]
 ; CHECK-NEXT:    ret i32 [[SUM]]
 ;
   %res = call i32 @callee(ptr %p, ptr %test_c)
@@ -91,19 +87,15 @@ define internal i32 @test_can_promote_1(ptr %p, ptr nocapture readonly %test_c)
   ret i32 %sum
 }
 
-; FIXME: We should perform ArgPromotion here!
-;
 ; This is called by multiple callers (@caller_safe_args_1, @caller_safe_args_2),
 ; from which we can prove that %test_c does not alias %p for any Call to the
 ; function, so we can promote it.
 ;
 define internal i32 @test_can_promote_2(ptr %p, ptr nocapture readonly %test_c) {
 ; CHECK-LABEL: define {{[^@]+}}@test_can_promote_2
-; CHECK-SAME: (ptr [[P:%.*]], ptr nocapture readonly [[TEST_C:%.*]]) {
-; CHECK-NEXT:    [[TEST_C_VAL:%.*]] = load i32, ptr [[TEST_C]], align 4
-; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_VAL]])
-; CHECK-NEXT:    [[LTEST_C:%.*]] = load i32, ptr [[TEST_C]], align 4
-; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[LTEST_C]], [[RES]]
+; CHECK-SAME: (ptr [[P:%.*]], i32 [[TEST_C_0_VAL:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @callee(ptr [[P]], i32 [[TEST_C_0_VAL]])
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[TEST_C_0_VAL]], [[RES]]
 ; CHECK-NEXT:    ret i32 [[SUM]]
 ;
   %res = call i32 @callee(ptr %p, ptr %test_c)
@@ -186,8 +178,10 @@ define i32 @caller_safe_args_1(i64 %n) {
 ; CHECK-NEXT:    [[CALLER_C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 5, ptr [[CALLER_C]], align 4
 ; CHECK-NEXT:    [[RES1:%.*]] = call i32 @test_cannot_promote_3(ptr [[P]], ptr [[CALLER_C]])
-; CHECK-NEXT:    [[RES2:%.*]] = call i32 @test_can_promote_1(ptr [[P]], ptr [[CALLER_C]])
-; CHECK-NEXT:    [[RES3:%.*]] = call i32 @test_can_promote_2(ptr [[P]], ptr [[CALLER_C]])
+; CHECK-NEXT:    [[CALLER_C_VAL:%.*]] = load i32, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[RES2:%.*]] = call i32 @test_can_promote_1(ptr [[P]], i32 [[CALLER_C_VAL]])
+; CHECK-NEXT:    [[CALLER_C_VAL1:%.*]] = load i32, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[RES3:%.*]] = call i32 @test_can_promote_2(ptr [[P]], i32 [[CALLER_C_VAL1]])
 ; CHECK-NEXT:    [[RES12:%.*]] = add i32 [[RES1]], [[RES2]]
 ; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES12]], [[RES3]]
 ; CHECK-NEXT:    ret i32 [[RES]]
@@ -215,7 +209,8 @@ define i32 @caller_safe_args_2(i64 %n, ptr %p) {
 ; CHECK-NEXT:    call void @memset(ptr [[P]], i64 0, i64 [[N]])
 ; CHECK-NEXT:    [[CALLER_C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 5, ptr [[CALLER_C]], align 4
-; CHECK-NEXT:    [[RES:%.*]] = call i32 @test_can_promote_2(ptr [[P]], ptr [[CALLER_C]])
+; CHECK-NEXT:    [[CALLER_C_VAL:%.*]] = load i32, ptr [[CALLER_C]], align 4
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @test_can_promote_2(ptr [[P]], i32 [[CALLER_C_VAL]])
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   call void @memset(ptr %p, i64 0, i64 %n)
diff --git a/llvm/test/Transforms/GlobalMerge/macho-sections.ll b/llvm/test/Transforms/GlobalMerge/macho-sections.ll
new file mode 100644
index 0000000000000..2d6f81de7eb85
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/macho-sections.ll
@@ -0,0 +1,31 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+; RUN: opt -passes='global-merge<max-offset=100>' -S -o - %s | FileCheck %s
+
+;; Check that we do _not_ merge globals which are in certain special
+;; sections under Mach-O.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-apple-macos11"
+
+; CHECK: @cfstring1 = private global i32 1, section "__DATA,__cfstring"
+@cfstring1 = private global i32 1, section "__DATA,__cfstring"
+; CHECK: @cfstring2 = private global i32 2, section "__DATA,__cfstring"
+@cfstring2 = private global i32 2, section "__DATA,__cfstring"
+; CHECK: @objcclassrefs1 = private global i32 3, section "__DATA,__objc_classrefs,regular,no_dead_strip"
+@objcclassrefs1 = private global i32 3, section "__DATA,__objc_classrefs,regular,no_dead_strip"
+; CHECK: @objcclassrefs2 = private global i32 4, section "__DATA,__objc_classrefs,regular,no_dead_strip"
+@objcclassrefs2 = private global i32 4, section "__DATA,__objc_classrefs,regular,no_dead_strip"
+; CHECK: @objcselrefs1 = private global i32 5, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip"
+@objcselrefs1 = private global i32 5, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip"
+; CHECK: @objcselrefs2 = private global i32 6, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip"
+@objcselrefs2 = private global i32 6, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip"
+
+define void @use() {
+  load ptr, ptr @cfstring1
+  load ptr, ptr @cfstring2
+  load ptr, ptr @objcclassrefs1
+  load ptr, ptr @objcclassrefs2
+  load ptr, ptr @objcselrefs1
+  load ptr, ptr @objcselrefs2
+  ret void
+}
diff --git a/llvm/test/Transforms/GlobalMerge/macho-symbols.ll b/llvm/test/Transforms/GlobalMerge/macho-symbols.ll
new file mode 100644
index 0000000000000..a3e99a55668ba
--- /dev/null
+++ b/llvm/test/Transforms/GlobalMerge/macho-symbols.ll
@@ -0,0 +1,28 @@
+; RUN: opt -global-merge -global-merge-max-offset=100 -S -o - %s | FileCheck %s
+; RUN: opt -passes='global-merge<max-offset=100>' -S -o - %s | FileCheck %s
+
+;; For Mach-O, we do not expect any alias symbols to be created for
+;; internal/private symbols by GlobalMerge.
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-apple-macos11"
+
+@a = private global i32 1
+@b = private global i32 2
+@c = internal global i32 3
+@d = internal global i32 4
+
+; CHECK: @_MergedGlobals = internal global <{ i32, i32, i32, i32 }> <{ i32 1, i32 2, i32 3, i32 4 }>, align 4
+; CHECK-NOT: alias
+
+define void @use() {
+  ; CHECK: load i32, ptr @_MergedGlobals,
+  %x = load i32, ptr @a
+  ; CHECK: load i32, ptr getelementptr inbounds (<{ i32, i32, i32, i32 }>, ptr @_MergedGlobals, i32 0, i32 1)
+  %y = load i32, ptr @b
+  ; CHECK: load i32, ptr getelementptr inbounds (<{ i32, i32, i32, i32 }>, ptr @_MergedGlobals, i32 0, i32 2)
+  %z1 = load i32, ptr @c
+  ; CHECK: load i32, ptr getelementptr inbounds (<{ i32, i32, i32, i32 }>, ptr @_MergedGlobals, i32 0, i32 3)
+  %z2 = load i32, ptr @d
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
index 7e9d12241ee36..ea6422e5ed8c2 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
@@ -4085,10 +4085,10 @@ declare <4 x i32> @llvm.amdgcn.struct.ptr.tbuffer.load.v4i32(ptr addrspace(8), i
 
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
@@ -4096,12 +4096,12 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8
 ; Check that the intrinsic remains unchanged in the presence of TFE or LWE
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 1, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 1, i32 0)
 ; CHECK-NEXT:    [[DATA_VEC:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA_VEC]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
   %data.vec = extractvalue {<4 x float>,i32} %data, 0
   %elt0 = extractelement <4 x float> %data.vec, i32 0
   ret float %elt0
@@ -4110,12 +4110,12 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr,
 ; Check that the intrinsic remains unchanged in the presence of TFE or LWE
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 2, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 2, i32 0)
 ; CHECK-NEXT:    [[DATA_VEC:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA_VEC]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
   %data.vec = extractvalue {<4 x float>,i32} %data, 0
   %elt0 = extractelement <4 x float> %data.vec, i32 0
   ret float %elt0
@@ -4123,194 +4123,194 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr,
 
 define amdgpu_ps float @extract_elt0_image_sample_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.2d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(float %s, float %t, float %r, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 0, float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(float %s, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1darray.f32.f32(i32 1, float [[S:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1darray.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 2, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 4, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 8, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 8, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = insertelement <2 x float> poison, float [[DATA]], i64 0
 ; CHECK-NEXT:    ret <2 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = insertelement <3 x float> poison, float [[DATA]], i64 0
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32.v8i32.v4i32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32.v8i32.v4i32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cl
@@ -4318,15 +4318,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt1_image_sample_cl_2darray_v4f32_f32(float %s, float %t, float %slice, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_image_sample_cl_2darray_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d
@@ -4334,15 +4334,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt2_image_sample_d_cube_v4f32_f32_f32(float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt2_image_sample_d_cube_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32(i32 4, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32.v8i32.v4i32(i32 4, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 2
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.cl
@@ -4350,15 +4350,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32.v8i32.v4i32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 3
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.l
@@ -4366,15 +4366,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.1d.f32.f32(i32 4, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.1d.f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32.v8i32.v4i32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <2 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b
@@ -4382,15 +4382,15 @@ declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32.v8i32.v4i32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.cl
@@ -4398,15 +4398,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, flo
 
 define amdgpu_ps <2 x float> @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32(i32 12, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32.v8i32.v4i32(i32 12, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
   ret <2 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.lz
@@ -4414,15 +4414,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps <2 x float> @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32(i32 10, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32.v8i32.v4i32(i32 10, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   ret <2 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd
@@ -4430,15 +4430,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i
 
 define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32(i32 14, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32.v8i32.v4i32(i32 14, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
   ret <3 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.cl
@@ -4446,78 +4446,78 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, fl
 
 define amdgpu_ps half @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 3
   ret half %elt0
 }
 
 define amdgpu_ps half @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 4, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 4, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 2
   ret half %elt0
 }
 
 define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 2, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 2, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 1
   ret half %elt0
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32.v8i32.v4i32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
   ret <4 x half> %res
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32.v8i32.v4i32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   ret <4 x half> %res
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = insertelement <4 x half> poison, half [[DATA]], i64 0
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
   ret <4 x half> %res
 }
 
 define amdgpu_ps half @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 0
   ret half %elt0
 }
 
-declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c
@@ -4525,15 +4525,15 @@ declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cl
@@ -4541,15 +4541,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cl_1d_v4f32_f32(float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cl_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d
@@ -4557,15 +4557,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.cl
@@ -4573,15 +4573,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.l
@@ -4589,15 +4589,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_l_1d_v4f32_f32(float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_l_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b
@@ -4605,15 +4605,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.cl
@@ -4621,15 +4621,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.lz
@@ -4637,15 +4637,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_lz_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_lz_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd
@@ -4653,15 +4653,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.cl
@@ -4669,15 +4669,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.o
@@ -4685,15 +4685,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cl.o
@@ -4701,15 +4701,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8
 
 define amdgpu_ps float @extract_elt0_image_sample_cl_o_1d_v4f32_f32(i32 %offset, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cl_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.o
@@ -4717,15 +4717,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.cl.o
@@ -4733,15 +4733,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.l.o
@@ -4749,15 +4749,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_l_o_1d_v4f32_f32(i32 %offset, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_l_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.o
@@ -4765,15 +4765,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.cl.o
@@ -4781,15 +4781,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.lz.o
@@ -4797,15 +4797,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_lz_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_lz_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.o
@@ -4813,15 +4813,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.cl.o
@@ -4829,15 +4829,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, fl
 
 define amdgpu_ps float @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.o
@@ -4845,15 +4845,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cl.o
@@ -4861,15 +4861,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.o
@@ -4877,15 +4877,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.cl.o
@@ -4893,15 +4893,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.l.o
@@ -4909,15 +4909,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32
 
 define amdgpu_ps float @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.o
@@ -4925,15 +4925,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.cl.o
@@ -4941,15 +4941,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.lz.o
@@ -4957,15 +4957,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32
 
 define amdgpu_ps float @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.o
@@ -4973,15 +4973,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.cl.o
@@ -4989,15 +4989,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4
@@ -5007,16 +5007,16 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i3
 
 define amdgpu_ps float @extract_elt0_image_gather4_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.cl
@@ -5024,16 +5024,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <
 
 define amdgpu_ps float @extract_elt0_image_gather4_cl_2d_v4f32_f32(float %s, float %t, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_cl_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.l
@@ -5041,16 +5041,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_l_2d_v4f32_f32(float %s, float %t, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_l_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b
@@ -5058,16 +5058,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(float %bias, float %s, float %t, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32.v4i32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.cl
@@ -5075,16 +5075,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.lz
@@ -5092,16 +5092,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_lz_2d_v4f32_f16(half %s, half %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_lz_2d_v4f32_f16(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32.v4i32(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.o
@@ -5109,16 +5109,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half,
 
 define amdgpu_ps float @extract_elt0_image_gather4_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.cl.o
@@ -5126,16 +5126,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.l.o
@@ -5143,16 +5143,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_l_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_l_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.o
@@ -5160,16 +5160,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.cl.o
@@ -5177,16 +5177,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, fl
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.lz.o
@@ -5194,16 +5194,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.o
@@ -5211,16 +5211,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.cl.o
@@ -5228,16 +5228,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.l.o
@@ -5245,16 +5245,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.b.o
@@ -5262,16 +5262,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.b.cl.o
@@ -5279,16 +5279,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.lz.o
@@ -5296,16 +5296,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i3
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.getlod
@@ -5313,15 +5313,15 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_getlod_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_getlod_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getlod.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getlod.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32.v8i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32.v8i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.load
@@ -5329,15 +5329,15 @@ declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>
 
 define amdgpu_ps float @extract_elt0_image_load_2dmsaa_v4f32_i32(i32 %s, i32 %t, i32 %sample, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_load_2dmsaa_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SAMPLE:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SAMPLE:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.load.mip
@@ -5345,15 +5345,15 @@ declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_load_mip_1d_v4f32_i32(i32 %s, i32 %mip, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_load_mip_1d_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.mip.1d.f32.i32(i32 1, i32 [[S:%.*]], i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.mip.1d.f32.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.getresinfo
@@ -5361,15 +5361,15 @@ declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x
 
 define amdgpu_ps float @extract_elt0_image_getresinfo_1d_v4f32_i32(i32 %mip, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_getresinfo_1d_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32(i32 1, i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32.v8i32(i32 1, i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32.v8i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; TFE / LWE
@@ -5377,30 +5377,30 @@ declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i3
 
 define amdgpu_ps float @extract_elt0_tfe_image_load_1d_v4f32i32_i32(i32 %s, <8 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_tfe_image_load_1d_v4f32i32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 1)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 1)
 ; CHECK-NEXT:    [[RGBA:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[RGBA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   %rgba = extractvalue { <4 x float>, i32 } %data, 0
   %elt0 = extractelement <4 x float> %rgba, i32 0
   ret float %elt0
 }
 
-declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #1
 
 define amdgpu_hs float @tfe_check_assert() #0 {
 ; CHECK-LABEL: @tfe_check_assert(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2d.f32.i32.v8i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
+  %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index af12367b9759c..66184405f3092 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -4084,10 +4084,10 @@ declare <4 x i32> @llvm.amdgcn.struct.ptr.tbuffer.load.v4i32(ptr addrspace(8), i
 
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
@@ -4095,12 +4095,12 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8
 ; Check that the intrinsic remains unchanged in the presence of TFE or LWE
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 1, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 1, i32 0)
 ; CHECK-NEXT:    [[DATA_VEC:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA_VEC]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
   %data.vec = extractvalue {<4 x float>,i32} %data, 0
   %elt0 = extractelement <4 x float> %data.vec, i32 0
   ret float %elt0
@@ -4109,12 +4109,12 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr,
 ; Check that the intrinsic remains unchanged in the presence of TFE or LWE
 define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 2, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float [[VADDR:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 2, i32 0)
 ; CHECK-NEXT:    [[DATA_VEC:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA_VEC]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
   %data.vec = extractvalue {<4 x float>,i32} %data, 0
   %elt0 = extractelement <4 x float> %data.vec, i32 0
   ret float %elt0
@@ -4122,194 +4122,194 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr,
 
 define amdgpu_ps float @extract_elt0_image_sample_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.2d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(float %s, float %t, float %r, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 0, float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(float %s, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1darray.f32.f32(i32 1, float [[S:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1darray.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 2, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 4, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 8, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 8, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = insertelement <2 x float> poison, float [[DATA]], i64 0
 ; CHECK-NEXT:    ret <2 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 0, i32 1>
   ret <2 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = insertelement <3 x float> poison, float [[DATA]], i64 0
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32.v8i32.v4i32(i32 5, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x float> [[DATA]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
 ; CHECK-NEXT:    ret <3 x float> [[SHUF]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32.v8i32.v4i32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
 define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32.v8i32.v4i32(i32 7, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
   ret <3 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cl
@@ -4317,15 +4317,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt1_image_sample_cl_2darray_v4f32_f32(float %s, float %t, float %slice, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_image_sample_cl_2darray_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d
@@ -4333,15 +4333,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt2_image_sample_d_cube_v4f32_f32_f32(float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt2_image_sample_d_cube_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32(i32 4, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32.v8i32.v4i32(i32 4, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 2
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.cl
@@ -4349,15 +4349,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32.v8i32.v4i32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 3
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.l
@@ -4365,15 +4365,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.1d.f32.f32(i32 4, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.1d.f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32.v8i32.v4i32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <2 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b
@@ -4381,15 +4381,15 @@ declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32.v8i32.v4i32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 1
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.cl
@@ -4397,15 +4397,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, flo
 
 define amdgpu_ps <2 x float> @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32(i32 12, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32.v8i32.v4i32(i32 12, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 2>
   ret <2 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.lz
@@ -4413,15 +4413,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps <2 x float> @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32(i32 10, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32.v8i32.v4i32(i32 10, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <2 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <2 x i32> <i32 1, i32 3>
   ret <2 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd
@@ -4429,15 +4429,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i
 
 define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32(i32 14, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32.v8i32.v4i32(i32 14, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret <3 x float> [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %shuf = shufflevector <4 x float> %data, <4 x float> poison, <3 x i32> <i32 1, i32 2, i32 3>
   ret <3 x float> %shuf
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.cl
@@ -4445,78 +4445,78 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, fl
 
 define amdgpu_ps half @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 8, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 3
   ret half %elt0
 }
 
 define amdgpu_ps half @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 4, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 4, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 2
   ret half %elt0
 }
 
 define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 2, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 2, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 1
   ret half %elt0
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32.v8i32.v4i32(i32 7, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <3 x half> [[DATA]], <3 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
   ret <4 x half> %res
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32.v8i32.v4i32(i32 3, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = shufflevector <2 x half> [[DATA]], <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   ret <4 x half> %res
 }
 
 define amdgpu_ps <4 x half> @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[RES:%.*]] = insertelement <4 x half> poison, half [[DATA]], i64 0
 ; CHECK-NEXT:    ret <4 x half> [[RES]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %res = shufflevector <4 x half> %data, <4 x half> poison, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
   ret <4 x half> %res
 }
 
 define amdgpu_ps half @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32.v8i32.v4i32(i32 1, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret half [[DATA]]
 ;
-  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x half> %data, i32 0
   ret half %elt0
 }
 
-declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c
@@ -4524,15 +4524,15 @@ declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cl
@@ -4540,15 +4540,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cl_1d_v4f32_f32(float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cl_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d
@@ -4556,15 +4556,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.cl
@@ -4572,15 +4572,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.l
@@ -4588,15 +4588,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_l_1d_v4f32_f32(float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_l_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b
@@ -4604,15 +4604,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.cl
@@ -4620,15 +4620,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.lz
@@ -4636,15 +4636,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_lz_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_lz_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd
@@ -4652,15 +4652,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.cl
@@ -4668,15 +4668,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32.v8i32.v4i32(i32 1, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.o
@@ -4684,15 +4684,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cl.o
@@ -4700,15 +4700,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8
 
 define amdgpu_ps float @extract_elt0_image_sample_cl_o_1d_v4f32_f32(i32 %offset, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cl_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.o
@@ -4716,15 +4716,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.d.cl.o
@@ -4732,15 +4732,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.l.o
@@ -4748,15 +4748,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_l_o_1d_v4f32_f32(i32 %offset, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_l_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.o
@@ -4764,15 +4764,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.b.cl.o
@@ -4780,15 +4780,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.lz.o
@@ -4796,15 +4796,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_lz_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_lz_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.o
@@ -4812,15 +4812,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.cd.cl.o
@@ -4828,15 +4828,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, fl
 
 define amdgpu_ps float @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.o
@@ -4844,15 +4844,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cl.o
@@ -4860,15 +4860,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.o
@@ -4876,15 +4876,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.d.cl.o
@@ -4892,15 +4892,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.l.o
@@ -4908,15 +4908,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32
 
 define amdgpu_ps float @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.o
@@ -4924,15 +4924,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.b.cl.o
@@ -4940,15 +4940,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, f
 
 define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.lz.o
@@ -4956,15 +4956,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32
 
 define amdgpu_ps float @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.o
@@ -4972,15 +4972,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample.c.cd.cl.o
@@ -4988,15 +4988,15 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4
@@ -5006,16 +5006,16 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i3
 
 define amdgpu_ps float @extract_elt0_image_gather4_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.cl
@@ -5023,16 +5023,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <
 
 define amdgpu_ps float @extract_elt0_image_gather4_cl_2d_v4f32_f32(float %s, float %t, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_cl_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32.v4i32(i32 2, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.l
@@ -5040,16 +5040,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_l_2d_v4f32_f32(float %s, float %t, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_l_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32.v4i32(i32 4, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b
@@ -5057,16 +5057,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(float %bias, float %s, float %t, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32.v4i32(i32 8, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32.v8i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.cl
@@ -5074,16 +5074,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32.v4i32(i32 1, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[FACE:%.*]], float [[CLAMP:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32.v8i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.lz
@@ -5091,16 +5091,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_lz_2d_v4f32_f16(half %s, half %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_lz_2d_v4f32_f16(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32.v4i32(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16.v8i32(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.o
@@ -5108,16 +5108,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half,
 
 define amdgpu_ps float @extract_elt0_image_gather4_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.cl.o
@@ -5125,16 +5125,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, f
 
 define amdgpu_ps float @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.l.o
@@ -5142,16 +5142,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_l_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_l_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.o
@@ -5159,16 +5159,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.b.cl.o
@@ -5176,16 +5176,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, fl
 
 define amdgpu_ps float @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.lz.o
@@ -5193,16 +5193,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.o
@@ -5210,16 +5210,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.cl.o
@@ -5227,16 +5227,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float,
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.l.o
@@ -5244,16 +5244,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.b.o
@@ -5261,16 +5261,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, floa
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.b.cl.o
@@ -5278,16 +5278,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32.v8i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.gather4.c.lz.o
@@ -5295,16 +5295,16 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i3
 
 define amdgpu_ps float @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 1, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[GATHER4R:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[DATA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.getlod
@@ -5312,15 +5312,15 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, flo
 
 define amdgpu_ps float @extract_elt0_image_getlod_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_image_getlod_1d_v4f32_f32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getlod.1d.f32.f32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getlod.1d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], <8 x i32> [[SAMPLER:%.*]], <4 x i32> [[RSRC:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32.v8i32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32.v8i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.load
@@ -5328,15 +5328,15 @@ declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>
 
 define amdgpu_ps float @extract_elt0_image_load_2dmsaa_v4f32_i32(i32 %s, i32 %t, i32 %sample, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_load_2dmsaa_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SAMPLE:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[SAMPLE:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.load.mip
@@ -5344,15 +5344,15 @@ declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32,
 
 define amdgpu_ps float @extract_elt0_image_load_mip_1d_v4f32_i32(i32 %s, i32 %mip, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_load_mip_1d_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.mip.1d.f32.i32(i32 1, i32 [[S:%.*]], i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.mip.1d.f32.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.getresinfo
@@ -5360,15 +5360,15 @@ declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x
 
 define amdgpu_ps float @extract_elt0_image_getresinfo_1d_v4f32_i32(i32 %mip, <8 x i32> inreg %sampler) #0 {
 ; CHECK-LABEL: @extract_elt0_image_getresinfo_1d_v4f32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32(i32 1, i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32.v8i32(i32 1, i32 [[MIP:%.*]], <8 x i32> [[SAMPLER:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+  %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32.v8i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; TFE / LWE
@@ -5376,30 +5376,30 @@ declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i3
 
 define amdgpu_ps float @extract_elt0_tfe_image_load_1d_v4f32i32_i32(i32 %s, <8 x i32> inreg %rsrc) #0 {
 ; CHECK-LABEL: @extract_elt0_tfe_image_load_1d_v4f32i32_i32(
-; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 1)
+; CHECK-NEXT:    [[DATA:%.*]] = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 1)
 ; CHECK-NEXT:    [[RGBA:%.*]] = extractvalue { <4 x float>, i32 } [[DATA]], 0
 ; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <4 x float> [[RGBA]], i64 0
 ; CHECK-NEXT:    ret float [[ELT0]]
 ;
-  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
   %rgba = extractvalue { <4 x float>, i32 } %data, 0
   %elt0 = extractelement <4 x float> %rgba, i32 0
   ret float %elt0
 }
 
-declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32, i32, <8 x i32>, i32, i32) #1
 
 define amdgpu_hs float @tfe_check_assert() #0 {
 ; CHECK-LABEL: @tfe_check_assert(
-; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
+; CHECK-NEXT:    [[DATA:%.*]] = call float @llvm.amdgcn.image.load.2d.f32.i32.v8i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
 ; CHECK-NEXT:    ret float [[DATA]]
 ;
-  %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
+  %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
   %elt0 = extractelement <4 x float> %data, i32 0
   ret float %elt0
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics-gfx8.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics-gfx8.ll
index 0cb9b4b84a556..4df170dc8f85c 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics-gfx8.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics-gfx8.ll
@@ -5,22 +5,22 @@
 ; llvm.amdgcn.image.sample a16 is disabled on pre-gfx9
 ; --------------------------------------------------------------------
 
-declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 define amdgpu_kernel void @image_sample_a16_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_1d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -29,13 +29,13 @@ define amdgpu_kernel void @image_sample_a16_2d(ptr addrspace(1) %out, <8 x i32>
 ; CHECK-LABEL: @image_sample_a16_2d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -45,14 +45,14 @@ define amdgpu_kernel void @image_sample_a16_3d(ptr addrspace(1) %out, <8 x i32>
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
 ; CHECK-NEXT:    [[R32:%.*]] = fpext half [[R:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 15, float [[S32]], float [[T32]], float [[R32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], float [[T32]], float [[R32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %r32 = fpext half %r to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -63,14 +63,14 @@ define amdgpu_kernel void @image_sample_a16_cube(ptr addrspace(1) %out, <8 x i32
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
 ; CHECK-NEXT:    [[FACE32:%.*]] = fpext half [[FACE:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32 15, float [[S32]], float [[T32]], float [[FACE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], float [[T32]], float [[FACE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %face32 = fpext half %face to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -79,13 +79,13 @@ define amdgpu_kernel void @image_sample_a16_1darray(ptr addrspace(1) %out, <8 x
 ; CHECK-LABEL: @image_sample_a16_1darray(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[SLICE32:%.*]] = fpext half [[SLICE:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 15, float [[S32]], float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %slice32 = fpext half %slice to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -95,14 +95,14 @@ define amdgpu_kernel void @image_sample_a16_2darray(ptr addrspace(1) %out, <8 x
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
 ; CHECK-NEXT:    [[SLICE32:%.*]] = fpext half [[SLICE:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32 15, float [[S32]], float [[T32]], float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32.v4i32(i32 15, float [[S32]], float [[T32]], float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %slice32 = fpext half %slice to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index fabf8ab51764b..e4f8bb133e6ce 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -3159,96 +3159,96 @@ define amdgpu_kernel void @permlane64_uniform(ptr addrspace(1) %out, i32 %src0)
 ; llvm.amdgcn.image.sample a16
 ; --------------------------------------------------------------------
 
-declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 define amdgpu_kernel void @image_sample_a16_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_3d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %r) {
 ; CHECK-LABEL: @image_sample_a16_3d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %r32 = fpext half %r to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3256,133 +3256,133 @@ define amdgpu_kernel void @image_sample_a16_3d(ptr addrspace(1) %out, <8 x i32>
 define amdgpu_kernel void @image_sample_a16_cube(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %face) {
 ;
 ; CHECK-LABEL: @image_sample_a16_cube(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[FACE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[FACE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %face32 = fpext half %face to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_1darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_1darray(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f16(i32 15, half [[S:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %slice32 = fpext half %slice to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_2darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_2darray(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %slice32 = fpext half %slice to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_c_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32(i32 15, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f16(i32 15, half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32(i32 15, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32(i32 15, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32(i32 15, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_b16_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s) {
 ; CHECK-LABEL: @image_sample_a16_b16_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float %bias32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3390,26 +3390,26 @@ define amdgpu_kernel void @image_sample_a16_b16_1d(ptr addrspace(1) %out, <8 x i
 define amdgpu_kernel void @image_sample_a16_b32_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, half %s) {
 ; CHECK-LABEL: @image_sample_a16_b32_1d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float %bias, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_b16_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_b16_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32(i32 15, float %bias32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3418,26 +3418,26 @@ define amdgpu_kernel void @image_sample_a16_b32_2d(ptr addrspace(1) %out, <8 x i
 ; CHECK-LABEL: @image_sample_a16_b32_2d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32(i32 15, float %bias, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_b16_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_b16_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias32, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3445,26 +3445,26 @@ define amdgpu_kernel void @image_sample_a16_c_b16_1d(ptr addrspace(1) %out, <8 x
 define amdgpu_kernel void @image_sample_a16_c_b32_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_b32_1d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_b16_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_c_b16_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32(i32 15, float %bias32, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3473,27 +3473,27 @@ define amdgpu_kernel void @image_sample_a16_c_b32_2d(ptr addrspace(1) %out, <8 x
 ; CHECK-LABEL: @image_sample_a16_c_b32_2d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[T32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_b16_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_b16_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 15, float %bias32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3502,20 +3502,20 @@ define amdgpu_kernel void @image_sample_a16_b32_cl_1d(ptr addrspace(1) %out, <8
 ; CHECK-LABEL: @image_sample_a16_b32_cl_1d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[CLAMP32:%.*]] = fpext half [[CLAMP:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_b16_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_b16_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3523,7 +3523,7 @@ define amdgpu_kernel void @image_sample_a16_b16_cl_2d(ptr addrspace(1) %out, <8
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32(i32 15, float %bias32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3533,28 +3533,28 @@ define amdgpu_kernel void @image_sample_a16_b32_cl_2d(ptr addrspace(1) %out, <8
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
 ; CHECK-NEXT:    [[CLAMP32:%.*]] = fpext half [[CLAMP:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[T32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S32]], float [[T32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32(i32 15, float %bias, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_b16_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_b16_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %bias32 = fpext half %bias to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias32, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3563,20 +3563,20 @@ define amdgpu_kernel void @image_sample_a16_c_b32_cl_1d(ptr addrspace(1) %out, <
 ; CHECK-LABEL: @image_sample_a16_c_b32_cl_1d(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[CLAMP32:%.*]] = fpext half [[CLAMP:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_b16_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %bias, float %zcompare, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_b16_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f16.f16(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[BIAS:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3584,7 +3584,7 @@ define amdgpu_kernel void @image_sample_a16_c_b16_cl_2d(ptr addrspace(1) %out, <
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32(i32 15, float %bias32, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias32, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -3594,35 +3594,35 @@ define amdgpu_kernel void @image_sample_a16_c_b32_cl_2d(ptr addrspace(1) %out, <
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[T32:%.*]] = fpext half [[T:%.*]] to float
 ; CHECK-NEXT:    [[CLAMP32:%.*]] = fpext half [[CLAMP:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[T32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S32]], float [[T32]], float [[CLAMP32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %bias, float %zcompare, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_d_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, half %s) {
 ; CHECK-LABEL: @image_sample_a16_d_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_d_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_d_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3632,14 +3632,14 @@ define amdgpu_kernel void @image_sample_a16_d_2d(ptr addrspace(1) %out, <8 x i32
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_d_3d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, half %s, half %t, half %r) {
 ; CHECK-LABEL: @image_sample_a16_d_3d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DRDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[DRDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DRDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[DRDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3652,28 +3652,28 @@ define amdgpu_kernel void @image_sample_a16_d_3d(ptr addrspace(1) %out, <8 x i32
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %r32 = fpext half %r to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %drdh32, float %dsdv32, float %dtdv32, float %drdv32, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %drdh32, float %dsdv32, float %dtdv32, float %drdv32, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_d_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_c_d_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3683,14 +3683,14 @@ define amdgpu_kernel void @image_sample_a16_c_d_2d(ptr addrspace(1) %out, <8 x i
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_d_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_d_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3698,14 +3698,14 @@ define amdgpu_kernel void @image_sample_a16_d_cl_1d(ptr addrspace(1) %out, <8 x
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_d_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_d_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3716,14 +3716,14 @@ define amdgpu_kernel void @image_sample_a16_d_cl_2d(ptr addrspace(1) %out, <8 x
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_d_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3731,14 +3731,14 @@ define amdgpu_kernel void @image_sample_a16_c_d_cl_1d(ptr addrspace(1) %out, <8
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_d_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3749,28 +3749,28 @@ define amdgpu_kernel void @image_sample_a16_c_d_cl_2d(ptr addrspace(1) %out, <8
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cd_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, half %s) {
 ; CHECK-LABEL: @image_sample_a16_cd_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cd_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_cd_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3780,28 +3780,28 @@ define amdgpu_kernel void @image_sample_a16_cd_2d(ptr addrspace(1) %out, <8 x i3
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cd_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_cd_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cd_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_c_cd_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3811,14 +3811,14 @@ define amdgpu_kernel void @image_sample_a16_c_cd_2d(ptr addrspace(1) %out, <8 x
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cd_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_cd_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3826,14 +3826,14 @@ define amdgpu_kernel void @image_sample_a16_cd_cl_1d(ptr addrspace(1) %out, <8 x
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_cd_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_cd_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f16.f16(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3844,14 +3844,14 @@ define amdgpu_kernel void @image_sample_a16_cd_cl_2d(ptr addrspace(1) %out, <8 x
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cd_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, half %s, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_cd_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], half [[S:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3859,14 +3859,14 @@ define amdgpu_kernel void @image_sample_a16_c_cd_cl_1d(ptr addrspace(1) %out, <8
   %dsdv32 = fpext half %dsdv to float
   %s32 = fpext half %s to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_cd_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %clamp) {
 ; CHECK-LABEL: @image_sample_a16_c_cd_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f16.f16(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f16.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -3877,118 +3877,118 @@ define amdgpu_kernel void @image_sample_a16_c_cd_cl_2d(ptr addrspace(1) %out, <8
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %clamp32 = fpext half %clamp to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %clamp32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_l_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %lod) {
 ; CHECK-LABEL: @image_sample_a16_l_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f16(i32 15, half [[S:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %lod32 = fpext half %lod to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32 15, float %s32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %lod) {
 ; CHECK-LABEL: @image_sample_a16_l_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %lod32 = fpext half %lod to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 15, float %s32, float %t32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_l_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %lod) {
 ; CHECK-LABEL: @image_sample_a16_c_l_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %lod32 = fpext half %lod to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t, half %lod) {
 ; CHECK-LABEL: @image_sample_a16_c_l_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], half [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %lod32 = fpext half %lod to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32 15, float %zcompare, float %s32, float %t32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %t32, float %lod32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_lz_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_lz_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_lz_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_lz_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_lz_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s) {
 ; CHECK-LABEL: @image_sample_a16_c_lz_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_lz_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_c_lz_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f16(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f16.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32(i32 15, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_V1(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_c_d_o_2darray_V1(
-; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f16.f16(i32 4, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f16.f16.v8i32.v4i32(i32 4, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store float [[RES]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -3999,14 +3999,14 @@ define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_V1(ptr addrspace(1) %o
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %slice32 = fpext half %slice to float
-  %res = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32(i32 4, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32.v8i32.v4i32(i32 4, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store float %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_V2(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %t, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_c_d_o_2darray_V2(
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f16(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f16.v8i32.v4i32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <2 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -4017,14 +4017,14 @@ define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_V2(ptr addrspace(1) %o
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %slice32 = fpext half %slice to float
-  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32.v8i32.v4i32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_const(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, half %s, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_c_d_o_2darray_const(
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f16(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half 0xH3400, half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f16.v8i32.v4i32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[S:%.*]], half 0xH3400, half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <2 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -4034,7 +4034,7 @@ define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_const(ptr addrspace(1)
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %slice32 = fpext half %slice to float
-  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float 0.25, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32.v8i32.v4i32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float 0.25, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4043,7 +4043,7 @@ define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_const_noopt(ptr addrsp
 ; CHECK-LABEL: @image_sample_a16_c_d_o_2darray_const_noopt(
 ; CHECK-NEXT:    [[S32:%.*]] = fpext half [[S:%.*]] to float
 ; CHECK-NEXT:    [[SLICE32:%.*]] = fpext half [[SLICE:%.*]] to float
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S32]], float 1.000000e+10, float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f32.v8i32.v4i32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S32]], float 1.000000e+10, float [[SLICE32]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <2 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -4053,19 +4053,19 @@ define amdgpu_kernel void @image_sample_a16_c_d_o_2darray_const_noopt(ptr addrsp
   %dtdv32 = fpext half %dtdv to float
   %s32 = fpext half %s to float
   %slice32 = fpext half %slice to float
-  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float 1.0e+10, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32.v8i32.v4i32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s32, float 1.0e+10, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_load_a16_mip_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i16 %s) {
 ; CHECK-LABEL: @image_load_a16_mip_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = zext i16 %s to i32
-  %res = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32 15, i32 %s32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4073,37 +4073,37 @@ define amdgpu_kernel void @image_load_a16_mip_1d(ptr addrspace(1) %out, <8 x i32
 define amdgpu_kernel void @image_load_a16_mip_1d_noopt(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i16 %s) {
 ; CHECK-LABEL: @image_load_a16_mip_1d_noopt(
 ; CHECK-NEXT:    [[S32:%.*]] = sext i16 [[S:%.*]] to i32
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 [[S32]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 [[S32]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = sext i16 %s to i32
-  %res = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32 15, i32 %s32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_load_a16_mip_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i16 %s, i16 %t) {
 ; CHECK-LABEL: @image_load_a16_mip_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16(i32 15, i16 [[S:%.*]], i16 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16.v8i32(i32 15, i16 [[S:%.*]], i16 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = zext i16 %s to i32
   %t32 = zext i16 %t to i32
-  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %s32, i32 %t32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32.v8i32(i32 15, i32 %s32, i32 %t32, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_load_a16_mip_2d_const(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i16 %s) {
 ; CHECK-LABEL: @image_load_a16_mip_2d_const(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16(i32 15, i16 [[S:%.*]], i16 -1, <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i16.v8i32(i32 15, i16 [[S:%.*]], i16 -1, <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = zext i16 %s to i32
-  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %s32, i32 65535, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32.v8i32(i32 15, i32 %s32, i32 65535, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4111,12 +4111,12 @@ define amdgpu_kernel void @image_load_a16_mip_2d_const(ptr addrspace(1) %out, <8
 define amdgpu_kernel void @image_load_a16_mip_2d_const_noopt(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i16 %s) {
 ; CHECK-LABEL: @image_load_a16_mip_2d_const_noopt(
 ; CHECK-NEXT:    [[S32:%.*]] = zext i16 [[S:%.*]] to i32
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 [[S32]], i32 65536, <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 15, i32 [[S32]], i32 65536, <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = zext i16 %s to i32
-  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %s32, i32 65536, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32.v8i32(i32 15, i32 %s32, i32 65536, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4127,20 +4127,20 @@ define amdgpu_kernel void @image_load_a16_mip_2d_const_noopt(ptr addrspace(1) %o
 
 define amdgpu_kernel void @image_sample_g16_d_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
 ; CHECK-LABEL: @image_sample_g16_d_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_d_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @image_sample_g16_d_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4148,14 +4148,14 @@ define amdgpu_kernel void @image_sample_g16_d_2d(ptr addrspace(1) %out, <8 x i32
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_d_3d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %drdh, half %dsdv, half %dtdv, half %drdv, float %s, float %t, float %r) {
 ; CHECK-LABEL: @image_sample_g16_d_3d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DRDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[DRDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DRDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], half [[DRDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4165,27 +4165,27 @@ define amdgpu_kernel void @image_sample_g16_d_3d(ptr addrspace(1) %out, <8 x i32
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
   %drdv32 = fpext half %drdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %drdh32, float %dsdv32, float %dtdv32, float %drdv32, float %s, float %t, float %r, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.3d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %drdh32, float %dsdv32, float %dtdv32, float %drdv32, float %s, float %t, float %r, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
 ; CHECK-LABEL: @image_sample_g16_c_d_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @image_sample_g16_c_d_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4193,27 +4193,27 @@ define amdgpu_kernel void @image_sample_g16_c_d_2d(ptr addrspace(1) %out, <8 x i
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_d_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_d_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_d_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_d_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4221,27 +4221,27 @@ define amdgpu_kernel void @image_sample_g16_d_cl_2d(ptr addrspace(1) %out, <8 x
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_c_d_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_c_d_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4249,27 +4249,27 @@ define amdgpu_kernel void @image_sample_g16_c_d_cl_2d(ptr addrspace(1) %out, <8
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_cd_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s) {
 ; CHECK-LABEL: @image_sample_g16_cd_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_cd_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @image_sample_g16_cd_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4277,27 +4277,27 @@ define amdgpu_kernel void @image_sample_g16_cd_2d(ptr addrspace(1) %out, <8 x i3
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_cd_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s) {
 ; CHECK-LABEL: @image_sample_g16_c_cd_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_cd_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @image_sample_g16_c_cd_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4305,27 +4305,27 @@ define amdgpu_kernel void @image_sample_g16_c_cd_2d(ptr addrspace(1) %out, <8 x
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_cd_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_cd_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32(i32 15, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_cd_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_cd_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f16.f32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4333,27 +4333,27 @@ define amdgpu_kernel void @image_sample_g16_cd_cl_2d(ptr addrspace(1) %out, <8 x
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_cd_cl_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_c_cd_cl_1d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %dsdh32 = fpext half %dsdh to float
   %dsdv32 = fpext half %dsdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dsdv32, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_cd_cl_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @image_sample_g16_c_cd_cl_2d(
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f16.f32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f16.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
@@ -4361,14 +4361,14 @@ define amdgpu_kernel void @image_sample_g16_c_cd_cl_2d(ptr addrspace(1) %out, <8
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_o_2darray_V1(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
 ; CHECK-LABEL: @image_sample_g16_c_d_o_2darray_V1(
-; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f16.f32(i32 4, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f16.f32.v8i32.v4i32(i32 4, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store float [[RES]], ptr addrspace(1) [[OUT:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -4376,14 +4376,14 @@ define amdgpu_kernel void @image_sample_g16_c_d_o_2darray_V1(ptr addrspace(1) %o
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32(i32 4, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32.v8i32.v4i32(i32 4, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store float %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_g16_c_d_o_2darray_V2(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %dsdh, half %dtdh, half %dsdv, half %dtdv, float %s, float %t, float %slice) {
 ; CHECK-LABEL: @image_sample_g16_c_d_o_2darray_V2(
-; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f16.f32.v8i32.v4i32(i32 6, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[DSDH:%.*]], half [[DTDH:%.*]], half [[DSDV:%.*]], half [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <2 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -4391,7 +4391,7 @@ define amdgpu_kernel void @image_sample_g16_c_d_o_2darray_V2(ptr addrspace(1) %o
   %dtdh32 = fpext half %dtdh to float
   %dsdv32 = fpext half %dsdv to float
   %dtdv32 = fpext half %dtdv to float
-  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32.v8i32.v4i32(i32 6, i32 %offset, float %zcompare, float %dsdh32, float %dtdh32, float %dsdv32, float %dtdv32, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <2 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4402,63 +4402,63 @@ define amdgpu_kernel void @image_sample_g16_c_d_o_2darray_V2(ptr addrspace(1) %o
 
 define amdgpu_kernel void @image_sample_a16_1d_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_1d_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_1d_nnan_ninf_nsz(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_1d_nnan_ninf_nsz(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan ninf nsz <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan ninf nsz <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call nnan ninf nsz <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan ninf nsz <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_1d_fast(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @image_sample_a16_1d_fast(
-; CHECK-NEXT:    [[RES:%.*]] = call fast <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call fast <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
-  %res = call fast <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call fast <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_2d_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
 ; CHECK-LABEL: @image_sample_a16_2d_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_3d_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %r) {
 ; CHECK-LABEL: @image_sample_a16_3d_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[R:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %r32 = fpext half %r to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %r32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4466,41 +4466,41 @@ define amdgpu_kernel void @image_sample_a16_3d_nnan(ptr addrspace(1) %out, <8 x
 define amdgpu_kernel void @image_sample_a16_cube_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %face) {
 ;
 ; CHECK-LABEL: @image_sample_a16_cube_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[FACE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[FACE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %face32 = fpext half %face to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %face32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_1darray_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_1darray_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f16(i32 15, half [[S:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %slice32 = fpext half %slice to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
 
 define amdgpu_kernel void @image_sample_a16_2darray_nnan(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t, half %slice) {
 ; CHECK-LABEL: @image_sample_a16_2darray_nnan(
-; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f16(i32 15, half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[RES:%.*]] = call nnan <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], half [[T:%.*]], half [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[RES]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %s32 = fpext half %s to float
   %t32 = fpext half %t to float
   %slice32 = fpext half %slice to float
-  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %res = call nnan <4 x float> @llvm.amdgcn.image.sample.2darray.v4f32.f32.v8i32.v4i32(i32 15, float %s32, float %t32, float %slice32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %res, ptr addrspace(1) %out
   ret void
 }
@@ -4509,26 +4509,26 @@ define amdgpu_kernel void @image_sample_a16_2darray_nnan(ptr addrspace(1) %out,
 ; llvm.amdgcn.image.sample l to lz
 ; --------------------------------------------------------------------
 
-declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
-declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2darray.v4f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32.v8i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2darray.v4f32.f32.v8i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 define amdgpu_kernel void @sample_l_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %lod) {
 ; CHECK-LABEL: @sample_l_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32 15, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4536,12 +4536,12 @@ main_body:
 define amdgpu_kernel void @sample_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @sample_l_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 15, float %s, float %t, float -0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, float -0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4549,12 +4549,12 @@ main_body:
 define amdgpu_kernel void @sample_c_l_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %lod) {
 ; CHECK-LABEL: @sample_c_l_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float -2.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float -2.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4562,12 +4562,12 @@ main_body:
 define amdgpu_kernel void @sample_c_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @sample_c_l_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32 15, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4575,12 +4575,12 @@ main_body:
 define amdgpu_kernel void @sample_l_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %lod) {
 ; CHECK-LABEL: @sample_l_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4588,12 +4588,12 @@ main_body:
 define amdgpu_kernel void @sample_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @sample_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4601,12 +4601,12 @@ main_body:
 define amdgpu_kernel void @sample_c_l_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %lod) {
 ; CHECK-LABEL: @sample_c_l_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4614,12 +4614,12 @@ main_body:
 define amdgpu_kernel void @sample_c_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @sample_c_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4627,12 +4627,12 @@ main_body:
 define amdgpu_kernel void @gather4_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @gather4_l_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 15, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32.v8i32(i32 15, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4640,12 +4640,12 @@ main_body:
 define amdgpu_kernel void @gather4_c_l_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @gather4_c_l_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32(i32 15, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.2d.v4f32.f32.v8i32(i32 15, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4653,12 +4653,12 @@ main_body:
 define amdgpu_kernel void @gather4_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @gather4_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32.v8i32(i32 15, i32 %offset, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4666,12 +4666,12 @@ main_body:
 define amdgpu_kernel void @gather4_c_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @gather4_c_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32.v8i32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4679,12 +4679,12 @@ main_body:
 define amdgpu_kernel void @gather4_c_l_o_2darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t, float %slice, float %lod) {
 ; CHECK-LABEL: @gather4_c_l_o_2darray(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2darray.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2darray.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[SLICE:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2darray.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float %slice, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2darray.v4f32.f32.v8i32(i32 15, i32 %offset, float %zcompare, float %s, float %t, float %slice, float 0.0, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4696,12 +4696,12 @@ main_body:
 define amdgpu_kernel void @load_mip_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s) {
 ; CHECK-LABEL: @load_mip_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32 15, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4709,12 +4709,12 @@ main_body:
 define amdgpu_kernel void @load_mip_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s, i32 %t) {
 ; CHECK-LABEL: @load_mip_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4722,12 +4722,12 @@ main_body:
 define amdgpu_kernel void @load_mip_3d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @load_mip_3d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4735,12 +4735,12 @@ main_body:
 define amdgpu_kernel void @load_mip_1darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s, i32 %t) {
 ; CHECK-LABEL: @load_mip_1darray(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4748,12 +4748,12 @@ main_body:
 define amdgpu_kernel void @load_mip_2darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @load_mip_2darray(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4761,12 +4761,12 @@ main_body:
 define amdgpu_kernel void @load_mip_cube(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @load_mip_cube(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %u, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4775,7 +4775,7 @@ main_body:
 define amdgpu_kernel void @store_mip_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
 ; CHECK-LABEL: @store_mip_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4786,7 +4786,7 @@ main_body:
 define amdgpu_kernel void @store_mip_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
 ; CHECK-LABEL: @store_mip_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.2d.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4797,7 +4797,7 @@ main_body:
 define amdgpu_kernel void @store_mip_3d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @store_mip_3d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.3d.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.3d.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4808,7 +4808,7 @@ main_body:
 define amdgpu_kernel void @store_mip_1darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t) {
 ; CHECK-LABEL: @store_mip_1darray(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.1darray.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.1darray.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4819,7 +4819,7 @@ main_body:
 define amdgpu_kernel void @store_mip_2darray(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @store_mip_2darray(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.2darray.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.2darray.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4830,7 +4830,7 @@ main_body:
 define amdgpu_kernel void @store_mip_cube(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s, i32 %t, i32 %u) {
 ; CHECK-LABEL: @store_mip_cube(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    call void @llvm.amdgcn.image.store.cube.v4f32.i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; CHECK-NEXT:    call void @llvm.amdgcn.image.store.cube.v4f32.i32.v8i32(<4 x float> [[VDATA:%.*]], i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[U:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; CHECK-NEXT:    ret void
 ;
 main_body:
@@ -4838,49 +4838,49 @@ main_body:
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32.v8i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.2darray.v4f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32.v8i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 
 
-declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample bias zero
 ; --------------------------------------------------------------------
 
-declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f16(i32, i32, half, float, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.v8i32.v4i32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.v8i32.v4i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.v8i32.v4i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f16.v8i32.v4i32(i32, i32, half, float, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32.v8i32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32.v8i32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.v8i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 define amdgpu_kernel void @sample_b_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 ; CHECK-LABEL: @sample_b_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32(i32 15, float 0.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.v8i32.v4i32(i32 15, float 0.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4888,12 +4888,12 @@ main_body:
 define amdgpu_kernel void @sample_b_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; CHECK-LABEL: @sample_b_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32(i32 15, float -0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.v8i32.v4i32(i32 15, float -0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4901,12 +4901,12 @@ main_body:
 define amdgpu_kernel void @sample_c_b_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
 ; CHECK-LABEL: @sample_c_b_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32(i32 15, float -0.0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.v8i32.v4i32(i32 15, float -0.0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4914,12 +4914,12 @@ main_body:
 define amdgpu_kernel void @sample_c_b_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @sample_c_b_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32(i32 15, float 0.0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.v8i32.v4i32(i32 15, float 0.0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4927,12 +4927,12 @@ main_body:
 define amdgpu_kernel void @sample_b_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s) {
 ; CHECK-LABEL: @sample_b_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32(i32 15, i32 %offset, float 0.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float 0.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4940,12 +4940,12 @@ main_body:
 define amdgpu_kernel void @sample_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t) {
 ; CHECK-LABEL: @sample_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32(i32 15, i32 %offset, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4953,12 +4953,12 @@ main_body:
 define amdgpu_kernel void @sample_c_b_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s) {
 ; CHECK-LABEL: @sample_c_b_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32(i32 15, i32 %offset, float 0.0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float 0.0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4966,12 +4966,12 @@ main_body:
 define amdgpu_kernel void @sample_c_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @sample_c_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32(i32 15, i32 %offset, float 0.0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 %offset, float 0.0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4979,12 +4979,12 @@ main_body:
 define amdgpu_kernel void @gather4_b_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; CHECK-LABEL: @gather4_b_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32(i32 15, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.b.2d.v4f32.f32.v8i32(i32 15, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -4992,12 +4992,12 @@ main_body:
 define amdgpu_kernel void @gather4_c_b_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @gather4_c_b_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32(i32 15,  float 0.0, float %zcompare,float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.2d.v4f32.f32.v8i32(i32 15,  float 0.0, float %zcompare,float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5005,12 +5005,12 @@ main_body:
 define amdgpu_kernel void @gather4_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %s, float %t) {
 ; CHECK-LABEL: @gather4_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32(i32 15, i32 %offset, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.v8i32(i32 15, i32 %offset, float 0.0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5018,12 +5018,12 @@ main_body:
 define amdgpu_kernel void @gather4_c_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @gather4_c_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32(i32 15, i32 %offset,  float 0.0, float %zcompare,float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.v8i32(i32 15, i32 %offset,  float 0.0, float %zcompare,float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5031,12 +5031,12 @@ main_body:
 define amdgpu_kernel void @sample_c_b_o_a16_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, half %s, half %t) {
 ; CHECK-LABEL: @sample_c_b_o_a16_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f16(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f16.v8i32.v4i32(i32 15, i32 [[OFFSET:%.*]], float [[ZCOMPARE:%.*]], half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f16(i32 15, i32 %offset, half 0.0, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f16.v8i32.v4i32(i32 15, i32 %offset, half 0.0, float %zcompare, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5045,12 +5045,12 @@ main_body:
 define amdgpu_kernel void @sample_b_1d_pos(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 ; CHECK-LABEL: @sample_b_1d_pos(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float 1.000000e+00, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float 1.000000e+00, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32(i32 15, float 1.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.v8i32.v4i32(i32 15, float 1.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5059,12 +5059,12 @@ main_body:
 define amdgpu_kernel void @sample_b_1d_neg(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 ; CHECK-LABEL: @sample_b_1d_neg(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float -1.000000e+00, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float -1.000000e+00, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32(i32 15, float -1.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.v8i32.v4i32(i32 15, float -1.0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5073,13 +5073,13 @@ main_body:
 define amdgpu_kernel void @sample_b_1d_a16(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s) {
 ; CHECK-LABEL: @sample_b_1d_a16(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f16.v8i32.v4i32(i32 15, half [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
   %s32 = fpext half %s to float
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32(i32 15, float -0.0, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.v8i32.v4i32(i32 15, float -0.0, float %s32, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5091,12 +5091,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 ; CHECK-LABEL: @offset_sample_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5104,12 +5104,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32(i32 15, i32 0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5117,12 +5117,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
 ; CHECK-LABEL: @offset_sample_c_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5130,12 +5130,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_c_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5143,12 +5143,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32(i32 15, float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32 15, i32 0, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5156,12 +5156,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cl.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cl.o.2d.v4f32.f32(i32 15, i32 0, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5169,12 +5169,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5182,12 +5182,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cl.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.2d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5195,12 +5195,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_b_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s) {
 ; CHECK-LABEL: @offset_sample_b_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5208,12 +5208,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5221,12 +5221,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_b_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s) {
 ; CHECK-LABEL: @offset_sample_c_b_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5234,12 +5234,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_b_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_c_b_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5247,12 +5247,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_b_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_b_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5260,12 +5260,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_b_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_b_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5273,12 +5273,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_b_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_b_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5286,12 +5286,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_b_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %zcompare, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_b_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[BIAS:%.*]], float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5299,12 +5299,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_d_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s) {
 ; CHECK-LABEL: @offset_sample_d_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5312,12 +5312,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_d_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_d_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.d.o.2d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.d.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5325,12 +5325,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_d_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s) {
 ; CHECK-LABEL: @offset_sample_c_d_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5338,12 +5338,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_d_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_c_d_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.2d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5351,12 +5351,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_d_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_d_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5364,12 +5364,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_d_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_d_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5377,12 +5377,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_d_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_d_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5390,12 +5390,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_d_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_d_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5403,12 +5403,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cd_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s) {
 ; CHECK-LABEL: @offset_sample_cd_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5416,12 +5416,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cd_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_cd_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.o.2d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5429,12 +5429,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cd_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s) {
 ; CHECK-LABEL: @offset_sample_c_cd_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5442,12 +5442,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cd_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_c_cd_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.2d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5455,12 +5455,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cd_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_cd_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5468,12 +5468,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_cd_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_cd_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5481,12 +5481,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cd_cl_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_cd_cl_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DSDV:%.*]], float [[S:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5494,12 +5494,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_cd_cl_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp) {
 ; CHECK-LABEL: @offset_sample_c_cd_cl_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[DSDH:%.*]], float [[DTDH:%.*]], float [[DSDV:%.*]], float [[DTDV:%.*]], float [[S:%.*]], float [[T:%.*]], float [[CLAMP:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.2d.v4f32.f32.f32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %clamp, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5507,12 +5507,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_l_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %lod) {
 ; CHECK-LABEL: @offset_sample_l_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32(i32 15, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 0, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5520,12 +5520,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @offset_sample_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32(i32 15, i32 0, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.l.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5533,12 +5533,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_l_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %lod) {
 ; CHECK-LABEL: @offset_sample_c_l_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5546,12 +5546,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_l_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t, float %lod) {
 ; CHECK-LABEL: @offset_sample_c_l_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.l.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], float [[LOD:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %t, float %lod, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5559,12 +5559,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_lz_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
 ; CHECK-LABEL: @offset_sample_lz_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5572,12 +5572,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_lz_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_lz_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32(i32 15, i32 0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5585,12 +5585,12 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_lz_o_1d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s) {
 ; CHECK-LABEL: @offset_sample_c_lz_o_1d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
@@ -5598,56 +5598,56 @@ main_body:
 define amdgpu_kernel void @offset_sample_c_lz_o_2d(ptr addrspace(1) %out, <8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %s, float %t) {
 ; CHECK-LABEL: @offset_sample_c_lz_o_2d(
 ; CHECK-NEXT:  main_body:
-; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; CHECK-NEXT:    [[V:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[ZCOMPARE:%.*]], float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; CHECK-NEXT:    store <4 x float> [[V]], ptr addrspace(1) [[OUT:%.*]], align 16
 ; CHECK-NEXT:    ret void
 ;
 main_body:
-  %v = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32(i32 15, i32 0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+  %v = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32 15, i32 0, float %zcompare, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
   store <4 x float> %v, ptr addrspace(1) %out
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cl.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-
-declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
-declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.2d.v4f32.f32.f32.v8i32.v4i32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.2d.v4f32.f32.v8i32.v4i32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.is.shared
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
index 65c89618e20ba..e7a36e805a012 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-simplify-image-buffer-stores.ll
@@ -7,41 +7,41 @@
 
 define amdgpu_ps void @image_store_1d_store_all_zeros(<8 x i32> inreg %rsrc, i32 %s) #0 {
 ; GCN-LABEL: @image_store_1d_store_all_zeros(
-; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32(float 0.000000e+00, i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32.v8i32(float 0.000000e+00, i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GCN-NEXT:    ret void
 ;
 ; GFX12-LABEL: @image_store_1d_store_all_zeros(
-; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32(float 0.000000e+00, i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32.v8i32(float 0.000000e+00, i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX12-NEXT:    ret void
 ;
 ; GFXUNKNOWN-LABEL: @image_store_1d_store_all_zeros(
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> zeroinitializer, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> zeroinitializer, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFXUNKNOWN-NEXT:    ret void
 ;
-  call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> zeroinitializer, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> zeroinitializer, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
 define amdgpu_ps void @image_store_1d_store_insert_zeros_at_end(<8 x i32> inreg %rsrc, float %vdata1, i32 %s) #0 {
 ; GCN-LABEL: @image_store_1d_store_insert_zeros_at_end(
-; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32(float [[VDATA1:%.*]], i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32.v8i32(float [[VDATA1:%.*]], i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GCN-NEXT:    ret void
 ;
 ; GFX12-LABEL: @image_store_1d_store_insert_zeros_at_end(
 ; GFX12-NEXT:    [[NEWVDATA4:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[VDATA1:%.*]], i64 0
-; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[NEWVDATA4]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[NEWVDATA4]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX12-NEXT:    ret void
 ;
 ; GFXUNKNOWN-LABEL: @image_store_1d_store_insert_zeros_at_end(
 ; GFXUNKNOWN-NEXT:    [[NEWVDATA4:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[VDATA1:%.*]], i64 0
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[NEWVDATA4]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[NEWVDATA4]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFXUNKNOWN-NEXT:    ret void
 ;
   %newvdata1 = insertelement <4 x float> undef, float %vdata1, i32 0
   %newvdata2 = insertelement <4 x float> %newvdata1, float 0.0, i32 1
   %newvdata3 = insertelement <4 x float> %newvdata2, float 0.0, i32 2
   %newvdata4 = insertelement <4 x float> %newvdata3, float 0.0, i32 3
-  call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %newvdata4, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> %newvdata4, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
@@ -49,26 +49,26 @@ define amdgpu_ps void @image_store_mip_1d_store_insert_zeros_at_end(<8 x i32> in
 ; GCN-LABEL: @image_store_mip_1d_store_insert_zeros_at_end(
 ; GCN-NEXT:    [[TMP1:%.*]] = insertelement <3 x float> <float 0.000000e+00, float poison, float poison>, float [[VDATA1:%.*]], i64 1
 ; GCN-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> [[TMP1]], float [[VDATA2:%.*]], i64 2
-; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32(<3 x float> [[TMP2]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32.v8i32(<3 x float> [[TMP2]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GCN-NEXT:    ret void
 ;
 ; GFX12-LABEL: @image_store_mip_1d_store_insert_zeros_at_end(
 ; GFX12-NEXT:    [[TMP1:%.*]] = insertelement <3 x float> <float 0.000000e+00, float poison, float poison>, float [[VDATA1:%.*]], i64 1
 ; GFX12-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> [[TMP1]], float [[VDATA2:%.*]], i64 2
-; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32(<3 x float> [[TMP2]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32.v8i32(<3 x float> [[TMP2]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX12-NEXT:    ret void
 ;
 ; GFXUNKNOWN-LABEL: @image_store_mip_1d_store_insert_zeros_at_end(
 ; GFXUNKNOWN-NEXT:    [[TMP1:%.*]] = insertelement <4 x float> <float 0.000000e+00, float poison, float poison, float 0.000000e+00>, float [[VDATA1:%.*]], i64 1
 ; GFXUNKNOWN-NEXT:    [[NEWVDATA4:%.*]] = insertelement <4 x float> [[TMP1]], float [[VDATA2:%.*]], i64 2
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[NEWVDATA4]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[NEWVDATA4]], i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFXUNKNOWN-NEXT:    ret void
 ;
   %newvdata1 = insertelement <4 x float> undef, float 0.0, i32 0
   %newvdata2 = insertelement <4 x float> %newvdata1, float %vdata1, i32 1
   %newvdata3 = insertelement <4 x float> %newvdata2, float %vdata2, i32 2
   %newvdata4 = insertelement <4 x float> %newvdata3, float 0.0, i32 3
-  call void @llvm.amdgcn.image.store.mip.1d.v4f32.i32(<4 x float> %newvdata4, i32 7, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
+  call void @llvm.amdgcn.image.store.mip.1d.v4f32.i32.v8i32(<4 x float> %newvdata4, i32 7, i32 %s, i32 0, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
@@ -148,39 +148,39 @@ define amdgpu_ps void @struct_tbuffer_store_insert_undefs(<4 x i32> inreg %a, fl
 define amdgpu_ps void @image_store_1d_store_shufflevector_same(<8 x i32> inreg %rsrc, <4 x float> %vdata1, i32 %s) #0 {
 ; GCN-LABEL: @image_store_1d_store_shufflevector_same(
 ; GCN-NEXT:    [[DATA:%.*]] = shufflevector <4 x float> [[VDATA1:%.*]], <4 x float> poison, <4 x i32> zeroinitializer
-; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[DATA]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[DATA]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GCN-NEXT:    ret void
 ;
 ; GFX12-LABEL: @image_store_1d_store_shufflevector_same(
 ; GFX12-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[VDATA1:%.*]], i64 0
-; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32(float [[TMP1]], i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.f32.i32.v8i32(float [[TMP1]], i32 1, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX12-NEXT:    ret void
 ;
 ; GFXUNKNOWN-LABEL: @image_store_1d_store_shufflevector_same(
 ; GFXUNKNOWN-NEXT:    [[DATA:%.*]] = shufflevector <4 x float> [[VDATA1:%.*]], <4 x float> poison, <4 x i32> zeroinitializer
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> [[DATA]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> [[DATA]], i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFXUNKNOWN-NEXT:    ret void
 ;
   %data = shufflevector <4 x float> %vdata1, <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
-  call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %data, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> %data, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
 define amdgpu_ps void @image_store_1d_store_shufflevector(<8 x i32> inreg %rsrc, float %vdata1, i32 %s) #0 {
 ; GCN-LABEL: @image_store_1d_store_shufflevector(
-; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00, float 2.000000e+00>, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GCN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00, float 2.000000e+00>, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GCN-NEXT:    ret void
 ;
 ; GFX12-LABEL: @image_store_1d_store_shufflevector(
-; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32(<3 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00>, i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX12-NEXT:    call void @llvm.amdgcn.image.store.1d.v3f32.i32.v8i32(<3 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00>, i32 7, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX12-NEXT:    ret void
 ;
 ; GFXUNKNOWN-LABEL: @image_store_1d_store_shufflevector(
-; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00, float 2.000000e+00>, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFXUNKNOWN-NEXT:    call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> <float 2.000000e+00, float 2.000000e+00, float 5.000000e+00, float 2.000000e+00>, i32 15, i32 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFXUNKNOWN-NEXT:    ret void
 ;
   %data = shufflevector <4 x float> <float 2.0, float 1.0, float 2.0, float 5.0>, <4 x float> poison, <4 x i32> <i32 0, i32 0, i32 3, i32 2>
-  call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %data, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  call void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float> %data, i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
 }
 
@@ -280,20 +280,20 @@ declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>,
 declare void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #2
 declare void @llvm.amdgcn.struct.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.tbuffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32) #0
-declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.3d.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.cube.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.1darray.v4f32.i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.2darray.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.2dmsaa.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.2darraymsaa.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
-declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.1d.v4f32.i32.v8i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.2d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.3d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.cube.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.1darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.2darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.2dmsaa.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.2darraymsaa.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.1d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.2d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.3d.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.cube.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.1darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
+declare void @llvm.amdgcn.image.store.mip.2darray.v4f32.i32.v8i32(<4 x float>, i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #0
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind writeonly }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/image-d16.ll b/llvm/test/Transforms/InstCombine/AMDGPU/image-d16.ll
index d8d3c25688367..30431ad724843 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/image-d16.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/image-d16.ll
@@ -7,17 +7,17 @@
 define amdgpu_ps half @image_sample_2d_fptrunc_to_d16(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; GFX7-LABEL: @image_sample_2d_fptrunc_to_d16(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[TEX:%.*]] = call float @llvm.amdgcn.image.sample.lz.2d.f32.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX7-NEXT:    [[TEX:%.*]] = call float @llvm.amdgcn.image.sample.lz.2d.f32.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX7-NEXT:    [[TEX_HALF:%.*]] = fptrunc float [[TEX]] to half
 ; GFX7-NEXT:    ret half [[TEX_HALF]]
 ;
 ; GFX81PLUS-LABEL: @image_sample_2d_fptrunc_to_d16(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[TEX:%.*]] = call half @llvm.amdgcn.image.sample.lz.2d.f16.f32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[TEX:%.*]] = call half @llvm.amdgcn.image.sample.lz.2d.f16.f32.v8i32.v4i32(i32 1, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX81PLUS-NEXT:    ret half [[TEX]]
 ;
 main_body:
-  %tex = call float @llvm.amdgcn.image.sample.lz.2d.f32.f32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  %tex = call float @llvm.amdgcn.image.sample.lz.2d.f32.f32.v8i32.v4i32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   %tex_half = fptrunc float %tex to half
   ret half %tex_half
 }
@@ -25,7 +25,7 @@ main_body:
 define amdgpu_ps half @image_sample_2d_v2f32(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; GFX7-LABEL: @image_sample_2d_v2f32(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[TEX:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.2d.v2f32.f32(i32 3, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX7-NEXT:    [[TEX:%.*]] = call <2 x float> @llvm.amdgcn.image.sample.lz.2d.v2f32.f32.v8i32.v4i32(i32 3, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX7-NEXT:    [[TEX_2_HALF:%.*]] = fptrunc <2 x float> [[TEX]] to <2 x half>
 ; GFX7-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <2 x half> [[TEX_2_HALF]], i64 0
 ; GFX7-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <2 x half> [[TEX_2_HALF]], i64 1
@@ -34,14 +34,14 @@ define amdgpu_ps half @image_sample_2d_v2f32(<8 x i32> inreg %rsrc, <4 x i32> in
 ;
 ; GFX81PLUS-LABEL: @image_sample_2d_v2f32(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.lz.2d.v2f16.f32(i32 3, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <2 x half> @llvm.amdgcn.image.sample.lz.2d.v2f16.f32.v8i32.v4i32(i32 3, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <2 x half> [[TEX]], i64 0
 ; GFX81PLUS-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <2 x half> [[TEX]], i64 1
 ; GFX81PLUS-NEXT:    [[ADDF_SUM_0:%.*]] = fadd half [[TEX_HALF_0]], [[TEX_HALF_1]]
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_0]]
 ;
 main_body:
-  %tex = call <2 x float> @llvm.amdgcn.image.sample.lz.2d.v2f32.f32(i32 3, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  %tex = call <2 x float> @llvm.amdgcn.image.sample.lz.2d.v2f32.f32.v8i32.v4i32(i32 3, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   %tex_2_half = fptrunc <2 x float> %tex to <2 x half>
   %tex_half_0 = extractelement <2 x half> %tex_2_half, i64 0
   %tex_half_1 = extractelement <2 x half> %tex_2_half, i64 1
@@ -52,7 +52,7 @@ main_body:
 define amdgpu_ps half @image_sample_2d_v3f32(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; GFX7-LABEL: @image_sample_2d_v3f32(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[TEX:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.lz.2d.v3f32.f32(i32 7, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX7-NEXT:    [[TEX:%.*]] = call <3 x float> @llvm.amdgcn.image.sample.lz.2d.v3f32.f32.v8i32.v4i32(i32 7, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX7-NEXT:    [[TEX_3_HALF:%.*]] = fptrunc <3 x float> [[TEX]] to <3 x half>
 ; GFX7-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <3 x half> [[TEX_3_HALF]], i64 0
 ; GFX7-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <3 x half> [[TEX_3_HALF]], i64 1
@@ -63,7 +63,7 @@ define amdgpu_ps half @image_sample_2d_v3f32(<8 x i32> inreg %rsrc, <4 x i32> in
 ;
 ; GFX81PLUS-LABEL: @image_sample_2d_v3f32(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.lz.2d.v3f16.f32(i32 7, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <3 x half> @llvm.amdgcn.image.sample.lz.2d.v3f16.f32.v8i32.v4i32(i32 7, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <3 x half> [[TEX]], i64 0
 ; GFX81PLUS-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <3 x half> [[TEX]], i64 1
 ; GFX81PLUS-NEXT:    [[TEX_HALF_2:%.*]] = extractelement <3 x half> [[TEX]], i64 2
@@ -72,7 +72,7 @@ define amdgpu_ps half @image_sample_2d_v3f32(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_1]]
 ;
 main_body:
-  %tex = call <3 x float> @llvm.amdgcn.image.sample.lz.2d.v3f32.f32(i32 7, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  %tex = call <3 x float> @llvm.amdgcn.image.sample.lz.2d.v3f32.f32.v8i32.v4i32(i32 7, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   %tex_3_half = fptrunc <3 x float> %tex to <3 x half>
   %tex_half_0 = extractelement <3 x half> %tex_3_half, i64 0
   %tex_half_1 = extractelement <3 x half> %tex_3_half, i64 1
@@ -85,7 +85,7 @@ main_body:
 define amdgpu_ps half @image_sample_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
 ; GFX7-LABEL: @image_sample_2d_v4f32(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[TEX:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX7-NEXT:    [[TEX:%.*]] = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX7-NEXT:    [[TEX_4_HALF:%.*]] = fptrunc <4 x float> [[TEX]] to <4 x half>
 ; GFX7-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <4 x half> [[TEX_4_HALF]], i64 0
 ; GFX7-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <4 x half> [[TEX_4_HALF]], i64 1
@@ -98,7 +98,7 @@ define amdgpu_ps half @image_sample_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> in
 ;
 ; GFX81PLUS-LABEL: @image_sample_2d_v4f32(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <4 x half> @llvm.amdgcn.image.sample.lz.2d.v4f16.f32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <4 x half> @llvm.amdgcn.image.sample.lz.2d.v4f16.f32.v8i32.v4i32(i32 15, float [[S:%.*]], float [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <4 x half> [[TEX]], i64 0
 ; GFX81PLUS-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <4 x half> [[TEX]], i64 1
 ; GFX81PLUS-NEXT:    [[TEX_HALF_2:%.*]] = extractelement <4 x half> [[TEX]], i64 2
@@ -109,7 +109,7 @@ define amdgpu_ps half @image_sample_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_2]]
 ;
 main_body:
-  %tex = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32(i32 15, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  %tex = call <4 x float> @llvm.amdgcn.image.sample.lz.2d.v4f32.f32.v8i32.v4i32(i32 15, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   %tex_4_half = fptrunc <4 x float> %tex to <4 x half>
   %tex_half_0 = extractelement <4 x half> %tex_4_half, i64 0
   %tex_half_1 = extractelement <4 x half> %tex_4_half, i64 1
@@ -124,7 +124,7 @@ main_body:
 define amdgpu_ps half @image_gather4_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, half %s, half %t) {
 ; GFX7-LABEL: @image_gather4_2d_v4f32(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[TEX:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX7-NEXT:    [[TEX:%.*]] = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16.v8i32.v4i32(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX7-NEXT:    [[TEX_4_HALF:%.*]] = fptrunc <4 x float> [[TEX]] to <4 x half>
 ; GFX7-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <4 x half> [[TEX_4_HALF]], i64 0
 ; GFX7-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <4 x half> [[TEX_4_HALF]], i64 1
@@ -137,7 +137,7 @@ define amdgpu_ps half @image_gather4_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> i
 ;
 ; GFX81PLUS-LABEL: @image_gather4_2d_v4f32(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <4 x half> @llvm.amdgcn.image.gather4.2d.v4f16.f16(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[TEX:%.*]] = call <4 x half> @llvm.amdgcn.image.gather4.2d.v4f16.f16.v8i32.v4i32(i32 1, half [[S:%.*]], half [[T:%.*]], <8 x i32> [[RSRC:%.*]], <4 x i32> [[SAMP:%.*]], i1 false, i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[TEX_HALF_0:%.*]] = extractelement <4 x half> [[TEX]], i64 0
 ; GFX81PLUS-NEXT:    [[TEX_HALF_1:%.*]] = extractelement <4 x half> [[TEX]], i64 1
 ; GFX81PLUS-NEXT:    [[TEX_HALF_2:%.*]] = extractelement <4 x half> [[TEX]], i64 2
@@ -148,7 +148,7 @@ define amdgpu_ps half @image_gather4_2d_v4f32(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_2]]
 ;
 main_body:
-  %tex = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  %tex = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f16.v8i32.v4i32(i32 1, half %s, half %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
   %tex_4_half = fptrunc <4 x float> %tex to <4 x half>
   %tex_half_0 = extractelement <4 x half> %tex_4_half, i64 0
   %tex_half_1 = extractelement <4 x half> %tex_4_half, i64 1
@@ -162,22 +162,22 @@ main_body:
 
 define amdgpu_ps half @load_1d(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-LABEL: @load_1d(
-; GFX7-NEXT:    [[S_FLOAT:%.*]] = call float @llvm.amdgcn.image.load.1d.f32.i16(i32 1, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[S_FLOAT:%.*]] = call float @llvm.amdgcn.image.load.1d.f32.i16.v8i32(i32 1, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[S_HALF:%.*]] = fptrunc float [[S_FLOAT]] to half
 ; GFX7-NEXT:    ret half [[S_HALF]]
 ;
 ; GFX81PLUS-LABEL: @load_1d(
-; GFX81PLUS-NEXT:    [[S_FLOAT:%.*]] = call half @llvm.amdgcn.image.load.1d.f16.i16(i32 1, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[S_FLOAT:%.*]] = call half @llvm.amdgcn.image.load.1d.f16.i16.v8i32(i32 1, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    ret half [[S_FLOAT]]
 ;
-  %s_float = call float @llvm.amdgcn.image.load.1d.f32.i16(i32 1, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %s_float = call float @llvm.amdgcn.image.load.1d.f32.i16.v8i32(i32 1, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %s_half = fptrunc float %s_float to half
   ret half %s_half
 }
 
 define amdgpu_ps half @load_1d_v2(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-LABEL: @load_1d_v2(
-; GFX7-NEXT:    [[V2_FLOAT:%.*]] = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i16(i32 3, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V2_FLOAT:%.*]] = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i16.v8i32(i32 3, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V2_HALF:%.*]] = fptrunc <2 x float> [[V2_FLOAT]] to <2 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <2 x half> [[V2_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <2 x half> [[V2_HALF]], i64 1
@@ -185,13 +185,13 @@ define amdgpu_ps half @load_1d_v2(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-NEXT:    ret half [[ADDF_SUM_0]]
 ;
 ; GFX81PLUS-LABEL: @load_1d_v2(
-; GFX81PLUS-NEXT:    [[V2_FLOAT:%.*]] = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i16(i32 3, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V2_FLOAT:%.*]] = call <2 x half> @llvm.amdgcn.image.load.1d.v2f16.i16.v8i32(i32 3, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <2 x half> [[V2_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <2 x half> [[V2_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[ADDF_SUM_0:%.*]] = fadd half [[S0]], [[S1]]
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_0]]
 ;
-  %v2_float = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i16(i32 3, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %v2_float = call <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i16.v8i32(i32 3, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %v2_half = fptrunc <2 x float> %v2_float to <2 x half>
   %s0 = extractelement <2 x half> %v2_half, i64 0
   %s1 = extractelement <2 x half> %v2_half, i64 1
@@ -201,7 +201,7 @@ define amdgpu_ps half @load_1d_v2(i16 %s, <8 x i32> inreg %rsrc) {
 
 define amdgpu_ps half @load_1d_v3(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-LABEL: @load_1d_v3(
-; GFX7-NEXT:    [[V3_FLOAT:%.*]] = call <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i16(i32 7, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V3_FLOAT:%.*]] = call <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i16.v8i32(i32 7, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V3_HALF:%.*]] = fptrunc <3 x float> [[V3_FLOAT]] to <3 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <3 x half> [[V3_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <3 x half> [[V3_HALF]], i64 1
@@ -211,7 +211,7 @@ define amdgpu_ps half @load_1d_v3(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-NEXT:    ret half [[ADDF_SUM_1]]
 ;
 ; GFX81PLUS-LABEL: @load_1d_v3(
-; GFX81PLUS-NEXT:    [[V3_FLOAT:%.*]] = call <3 x half> @llvm.amdgcn.image.load.1d.v3f16.i16(i32 7, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V3_FLOAT:%.*]] = call <3 x half> @llvm.amdgcn.image.load.1d.v3f16.i16.v8i32(i32 7, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[S2:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 2
@@ -219,7 +219,7 @@ define amdgpu_ps half @load_1d_v3(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX81PLUS-NEXT:    [[ADDF_SUM_1:%.*]] = fadd half [[S2]], [[ADDF_SUM_0]]
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_1]]
 ;
-  %v3_float = call <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i16(i32 7, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %v3_float = call <3 x float> @llvm.amdgcn.image.load.1d.v3f32.i16.v8i32(i32 7, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %v3_half = fptrunc <3 x float> %v3_float to <3 x half>
   %s0 = extractelement <3 x half> %v3_half, i64 0
   %s1 = extractelement <3 x half> %v3_half, i64 1
@@ -231,7 +231,7 @@ define amdgpu_ps half @load_1d_v3(i16 %s, <8 x i32> inreg %rsrc) {
 
 define amdgpu_ps half @load_1d_v4(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-LABEL: @load_1d_v4(
-; GFX7-NEXT:    [[V4_FLOAT:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V4_FLOAT:%.*]] = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V4_HALF:%.*]] = fptrunc <4 x float> [[V4_FLOAT]] to <4 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <4 x half> [[V4_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <4 x half> [[V4_HALF]], i64 1
@@ -243,7 +243,7 @@ define amdgpu_ps half @load_1d_v4(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX7-NEXT:    ret half [[ADDF_SUM_2]]
 ;
 ; GFX81PLUS-LABEL: @load_1d_v4(
-; GFX81PLUS-NEXT:    [[V4_FLOAT:%.*]] = call <4 x half> @llvm.amdgcn.image.load.1d.v4f16.i16(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V4_FLOAT:%.*]] = call <4 x half> @llvm.amdgcn.image.load.1d.v4f16.i16.v8i32(i32 15, i16 [[S:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[S2:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 2
@@ -253,7 +253,7 @@ define amdgpu_ps half @load_1d_v4(i16 %s, <8 x i32> inreg %rsrc) {
 ; GFX81PLUS-NEXT:    [[ADDF_SUM_2:%.*]] = fadd half [[ADDF_SUM_0]], [[ADDF_SUM_1]]
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_2]]
 ;
-  %v4_float = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %v4_float = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
   %v4_half = fptrunc <4 x float> %v4_float to <4 x half>
   %s0 = extractelement <4 x half> %v4_half, i64 0
   %s1 = extractelement <4 x half> %v4_half, i64 1
@@ -268,17 +268,17 @@ define amdgpu_ps half @load_1d_v4(i16 %s, <8 x i32> inreg %rsrc) {
 define amdgpu_ps half @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
 ; GFX7-LABEL: @load_2dmsaa(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[S_FLOAT:%.*]] = call float @llvm.amdgcn.image.msaa.load.x.2dmsaa.f32.i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[S_FLOAT:%.*]] = call float @llvm.amdgcn.image.msaa.load.x.2dmsaa.f32.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[S_HALF:%.*]] = fptrunc float [[S_FLOAT]] to half
 ; GFX7-NEXT:    ret half [[S_HALF]]
 ;
 ; GFX81PLUS-LABEL: @load_2dmsaa(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[S_FLOAT:%.*]] = call half @llvm.amdgcn.image.msaa.load.x.2dmsaa.f16.i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[S_FLOAT:%.*]] = call half @llvm.amdgcn.image.msaa.load.x.2dmsaa.f16.i32.v8i32(i32 1, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    ret half [[S_FLOAT]]
 ;
 main_body:
-  %s_float = call float @llvm.amdgcn.image.msaa.load.x.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %s_float = call float @llvm.amdgcn.image.msaa.load.x.2dmsaa.f32.i32.v8i32(i32 1, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %s_half = fptrunc float %s_float to half
   ret half %s_half
 }
@@ -286,7 +286,7 @@ main_body:
 define amdgpu_ps half @load_2dmsaa_v2(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
 ; GFX7-LABEL: @load_2dmsaa_v2(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[V2_FLOAT:%.*]] = call <2 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f32.i32(i32 3, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V2_FLOAT:%.*]] = call <2 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f32.i32.v8i32(i32 3, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V2_HALF:%.*]] = fptrunc <2 x float> [[V2_FLOAT]] to <2 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <2 x half> [[V2_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <2 x half> [[V2_HALF]], i64 1
@@ -295,14 +295,14 @@ define amdgpu_ps half @load_2dmsaa_v2(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ;
 ; GFX81PLUS-LABEL: @load_2dmsaa_v2(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[V2_FLOAT:%.*]] = call <2 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f16.i32(i32 3, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V2_FLOAT:%.*]] = call <2 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f16.i32.v8i32(i32 3, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <2 x half> [[V2_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <2 x half> [[V2_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[ADDF_SUM_0:%.*]] = fadd half [[S0]], [[S1]]
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_0]]
 ;
 main_body:
-  %v2_float = call <2 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f32.i32(i32 3, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %v2_float = call <2 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v2f32.i32.v8i32(i32 3, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %v2_half = fptrunc <2 x float> %v2_float to <2 x half>
   %s0 = extractelement <2 x half> %v2_half, i64 0
   %s1 = extractelement <2 x half> %v2_half, i64 1
@@ -313,7 +313,7 @@ main_body:
 define amdgpu_ps half @load_2dmsaa_v3(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
 ; GFX7-LABEL: @load_2dmsaa_v3(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[V3_FLOAT:%.*]] = call <3 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f32.i32(i32 7, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V3_FLOAT:%.*]] = call <3 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f32.i32.v8i32(i32 7, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V3_HALF:%.*]] = fptrunc <3 x float> [[V3_FLOAT]] to <3 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <3 x half> [[V3_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <3 x half> [[V3_HALF]], i64 1
@@ -324,7 +324,7 @@ define amdgpu_ps half @load_2dmsaa_v3(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ;
 ; GFX81PLUS-LABEL: @load_2dmsaa_v3(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[V3_FLOAT:%.*]] = call <3 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f16.i32(i32 7, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V3_FLOAT:%.*]] = call <3 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f16.i32.v8i32(i32 7, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[S2:%.*]] = extractelement <3 x half> [[V3_FLOAT]], i64 2
@@ -333,7 +333,7 @@ define amdgpu_ps half @load_2dmsaa_v3(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_1]]
 ;
 main_body:
-  %v3_float = call <3 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f32.i32(i32 7, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %v3_float = call <3 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v3f32.i32.v8i32(i32 7, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %v3_half = fptrunc <3 x float> %v3_float to <3 x half>
   %s0 = extractelement <3 x half> %v3_half, i64 0
   %s1 = extractelement <3 x half> %v3_half, i64 1
@@ -346,7 +346,7 @@ main_body:
 define amdgpu_ps half @load_2dmsaa_v4(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
 ; GFX7-LABEL: @load_2dmsaa_v4(
 ; GFX7-NEXT:  main_body:
-; GFX7-NEXT:    [[V4_FLOAT:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f32.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX7-NEXT:    [[V4_FLOAT:%.*]] = call <4 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f32.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX7-NEXT:    [[V4_HALF:%.*]] = fptrunc <4 x float> [[V4_FLOAT]] to <4 x half>
 ; GFX7-NEXT:    [[S0:%.*]] = extractelement <4 x half> [[V4_HALF]], i64 0
 ; GFX7-NEXT:    [[S1:%.*]] = extractelement <4 x half> [[V4_HALF]], i64 1
@@ -359,7 +359,7 @@ define amdgpu_ps half @load_2dmsaa_v4(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ;
 ; GFX81PLUS-LABEL: @load_2dmsaa_v4(
 ; GFX81PLUS-NEXT:  main_body:
-; GFX81PLUS-NEXT:    [[V4_FLOAT:%.*]] = call <4 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f16.i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
+; GFX81PLUS-NEXT:    [[V4_FLOAT:%.*]] = call <4 x half> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f16.i32.v8i32(i32 15, i32 [[S:%.*]], i32 [[T:%.*]], i32 [[FRAGID:%.*]], <8 x i32> [[RSRC:%.*]], i32 0, i32 0)
 ; GFX81PLUS-NEXT:    [[S0:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 0
 ; GFX81PLUS-NEXT:    [[S1:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 1
 ; GFX81PLUS-NEXT:    [[S2:%.*]] = extractelement <4 x half> [[V4_FLOAT]], i64 2
@@ -370,7 +370,7 @@ define amdgpu_ps half @load_2dmsaa_v4(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ; GFX81PLUS-NEXT:    ret half [[ADDF_SUM_2]]
 ;
 main_body:
-  %v4_float = call <4 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
+  %v4_float = call <4 x float> @llvm.amdgcn.image.msaa.load.x.2dmsaa.v4f32.i32.v8i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 0, i32 0)
   %v4_half = fptrunc <4 x float> %v4_float to <4 x half>
   %s0 = extractelement <4 x half> %v4_half, i64 0
   %s1 = extractelement <4 x half> %v4_half, i64 1
diff --git a/llvm/test/Transforms/LICM/hoist-binop.ll b/llvm/test/Transforms/LICM/hoist-binop.ll
index a840e24757884..74e2b7a2caf4a 100644
--- a/llvm/test/Transforms/LICM/hoist-binop.ll
+++ b/llvm/test/Transforms/LICM/hoist-binop.ll
@@ -437,17 +437,17 @@ loop:
   br label %loop
 }
 
-; Don't hoist floating-point ops, even if they are associative. This would be
-; valid, but is currently disabled.
-define void @fadd(float %c1, float %c2) {
-; CHECK-LABEL: @fadd(
+; The simple case. Hoist if fast is present on both instructions.
+define void @fadd_fast(float %c1, float %c2) {
+; CHECK-LABEL: @fadd_fast(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fadd fast float [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd fast float [[INDEX]], [[C1:%.*]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd fast float [[INDEX]], [[C1]]
 ; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = fadd fast float [[STEP_ADD]], [[C2:%.*]]
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fadd fast float [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
@@ -461,6 +461,226 @@ loop:
   br label %loop
 }
 
+; The simple case. Hoist if fast is present on both instructions.
+define void @fmul_fast(float %c1, float %c2) {
+; CHECK-LABEL: @fmul_fast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fmul fast float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fmul fast float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fmul fast float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fmul fast float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fmul fast float %step.add, %c2
+  br label %loop
+}
+
+; The minimum case.
+; Hoist if reasassoc and nsz are present on both instructions.
+define void @fadd_reassoc_nsz(float %c1, float %c2) {
+; CHECK-LABEL: @fadd_reassoc_nsz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fadd reassoc nsz float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd reassoc nsz float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fadd reassoc nsz float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fadd reassoc nsz float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fadd reassoc nsz float %step.add, %c2
+  br label %loop
+}
+
+; The minimum case.
+; Hoist if reasassoc and nsz are present on both instructions.
+define void @fmul_reassoc_nsz(float %c1, float %c2) {
+; CHECK-LABEL: @fmul_reassoc_nsz(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fmul reassoc nsz float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fmul reassoc nsz float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fmul reassoc nsz float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fmul reassoc nsz float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fmul reassoc nsz float %step.add, %c2
+  br label %loop
+}
+
+; Don't hoist if both reassoc and nsz aren't present on both instructions.
+define void @fadd_nonassoc(float %c1, float %c2) {
+; CHECK-LABEL: @fadd_nonassoc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd reassoc float [[INDEX]], [[C1:%.*]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = fadd reassoc nsz float [[STEP_ADD]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fadd reassoc float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fadd reassoc nsz float %step.add, %c2
+  br label %loop
+}
+
+; Don't hoist if both reassoc and nsz aren't present on both instructions.
+define void @fmul_noassoc(float %c1, float %c2) {
+; CHECK-LABEL: @fmul_noassoc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fmul reassoc nsz float [[INDEX]], [[C1:%.*]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = fmul nsz float [[STEP_ADD]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fmul reassoc nsz float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fmul nsz float %step.add, %c2
+  br label %loop
+}
+
+; No intersection in flags present on both instructions,
+; except reassoc and nsz.
+define void @fadd_fmf_nointersect(float %c1, float %c2) {
+; CHECK-LABEL: @fadd_fmf_nointersect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fadd reassoc nsz float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd reassoc nnan nsz float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fadd reassoc nsz float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fadd reassoc nsz nnan float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fadd reassoc nsz ninf float %step.add, %c2
+  br label %loop
+}
+
+; No intersection in flags present on both instructions,
+; except reassoc and nsz.
+define void @fmul_fmf_nointersect(float %c1, float %c2) {
+; CHECK-LABEL: @fmul_fmf_nointersect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fmul reassoc nsz float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fmul reassoc nsz contract float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fmul reassoc nsz float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fmul reassoc nsz contract float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fmul reassoc nnan nsz float %step.add, %c2
+  br label %loop
+}
+
+; Non-empty intersection in flags present on both instructions,
+; including reassoc and nsz.
+define void @fadd_fmf_intersect(float %c1, float %c2) {
+; CHECK-LABEL: @fadd_fmf_intersect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fadd reassoc ninf nsz float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fadd reassoc nnan ninf nsz float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fadd reassoc ninf nsz float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fadd reassoc nnan nsz ninf float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fadd reassoc ninf nsz float %step.add, %c2
+  br label %loop
+}
+
+; Non-empty intersection in flags present on both instructions,
+; including reassoc and nsz.
+define void @fmul_fmf_intersect(float %c1, float %c2) {
+; CHECK-LABEL: @fmul_fmf_intersect(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = fmul reassoc nsz afn float [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = fmul reassoc nsz arcp afn float [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(float [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = fmul reassoc nsz afn float [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi float [ 0., %entry ], [ %index.next, %loop ]
+  %step.add = fmul reassoc afn nsz arcp float %index, %c1
+  call void @use(float %step.add)
+  %index.next = fmul reassoc nsz afn float %step.add, %c2
+  br label %loop
+}
+
 ; Don't hoist if the intermediate op has more than two uses. This is an
 ; heuristic that can be adjusted if warranted. Currently we are being
 ; conservative to minimise potential impact in code size.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
new file mode 100644
index 0000000000000..a617f5e4e4893
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -0,0 +1,104 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -force-vector-interleave=1 -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  iter.check:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP6]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP14]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP15]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP22]], 1
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP23]]
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[TMP24]], 1
+; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP25]]
+; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP16]], 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX4]]
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i32>, ptr [[TMP18]], align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD5]] to <vscale x 2 x i64>
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 2 x i64> [[TMP19]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> [[TMP20]], i32 1, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
+; CHECK-NEXT:    br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N7:%.*]] = icmp eq i64 [[N_MOD_VF2]], 0
+; CHECK-NEXT:    br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
new file mode 100644
index 0000000000000..716e3c4fbd7a2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-no-scalar-interleave.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=2 -force-vector-width=1 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+;; Make sure we don't interleave a histogram when vectorization is disabled.
+
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_forced_scalar_interleave'
+; CHECK: LV: Not interleaving without vectorization due to histogram operations.
+
+define void @simple_histogram_forced_scalar_interleave(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram_forced_scalar_interleave(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
new file mode 100644
index 0000000000000..02bbfa839e690
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-outerloop-scevaddrec.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Make sure we don't detect a histogram operation if the index address is
+;; loop invariant.
+; CHECK: LV: Checking for a histogram on:   store i32 %inc, ptr %gep.bucket, align 4
+; CHECK-NEXT: LV: Can't vectorize due to memory conflicts
+; CHECK-NEXT: LV: Not vectorizing: Cannot prove legality.
+
+define void @outer_loop_scevaddrec(ptr noalias %buckets, ptr readonly %indices, i64 %N, i64 %M) {
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %outer.iv
+  br label %inner.body
+
+inner.body:
+  %iv = phi i64 [ 0, %outer.header ], [ %iv.next, %inner.body ]
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %outer.latch, label %inner.body
+
+outer.latch:
+  %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+  %outer.exitcond = icmp eq i64 %outer.iv.next, %M
+  br i1 %outer.exitcond, label %outer.exit, label %outer.header
+
+outer.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
new file mode 100644
index 0000000000000..9ca4b20341e0e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=1 -max-dependences=2 -debug-only=loop-vectorize,loop-accesses -S 2>&1 | FileCheck %s
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=1 -debug-only=loop-vectorize,loop-accesses -S 2>&1 | FileCheck %s --check-prefix=NORMAL_DEP_LIMIT
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Check that we don't crash if LAA gives up on recording dependences and
+;; returns a null pointer.
+
+; CHECK-LABEL: LAA: Checking a loop in 'many_deps'
+; CHECK: Too many dependences, stopped recording
+; CHECK: LV: Can't vectorize due to memory conflicts
+; CHECK: LV: Not vectorizing: Cannot prove legality.
+
+define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %other, i64 %N) {
+; CHECK-LABEL: define void @many_deps(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], ptr [[OTHER:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; CHECK-NEXT:    [[GEP_OTHER:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV]]
+; CHECK-NEXT:    [[L_OTHER:%.*]] = load i32, ptr [[GEP_OTHER]], align 4
+; CHECK-NEXT:    [[ADD_OTHER:%.*]] = add i32 [[L_OTHER]], [[IV_TRUNC]]
+; CHECK-NEXT:    store i32 [[ADD_OTHER]], ptr [[GEP_OTHER]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+; NORMAL_DEP_LIMIT-LABEL: define void @many_deps(
+; NORMAL_DEP_LIMIT-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], ptr [[OTHER:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; NORMAL_DEP_LIMIT-NEXT:  entry:
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
+; NORMAL_DEP_LIMIT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; NORMAL_DEP_LIMIT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; NORMAL_DEP_LIMIT:       vector.memcheck:
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP3:%.*]] = shl i64 [[N]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ARRAY]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[OTHER]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[INDICES]], i64 [[TMP3]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[ARRAY]], [[SCEVGEP1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[OTHER]], [[SCEVGEP]]
+; NORMAL_DEP_LIMIT-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND03:%.*]] = icmp ult ptr [[ARRAY]], [[SCEVGEP2]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND14:%.*]] = icmp ult ptr [[INDICES]], [[SCEVGEP]]
+; NORMAL_DEP_LIMIT-NEXT:    [[FOUND_CONFLICT5:%.*]] = and i1 [[BOUND03]], [[BOUND14]]
+; NORMAL_DEP_LIMIT-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND06:%.*]] = icmp ult ptr [[OTHER]], [[SCEVGEP2]]
+; NORMAL_DEP_LIMIT-NEXT:    [[BOUND17:%.*]] = icmp ult ptr [[INDICES]], [[SCEVGEP1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[FOUND_CONFLICT8:%.*]] = and i1 [[BOUND06]], [[BOUND17]]
+; NORMAL_DEP_LIMIT-NEXT:    [[CONFLICT_RDX9:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT8]]
+; NORMAL_DEP_LIMIT-NEXT:    br i1 [[CONFLICT_RDX9]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
+; NORMAL_DEP_LIMIT:       vector.ph:
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP4]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
+; NORMAL_DEP_LIMIT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP6]] to i32
+; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
+; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NORMAL_DEP_LIMIT-NEXT:    br label [[FOR_BODY:%.*]]
+; NORMAL_DEP_LIMIT:       vector.body:
+; NORMAL_DEP_LIMIT-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NORMAL_DEP_LIMIT-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP7]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
+; NORMAL_DEP_LIMIT-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[GEP_INDICES]], align 4, !alias.scope [[META0:![0-9]+]]
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP11]]
+; NORMAL_DEP_LIMIT-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP12]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT:    store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP13]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META5:![0-9]+]]
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV]]
+; NORMAL_DEP_LIMIT-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4, !alias.scope [[META7:![0-9]+]], !noalias [[META0]]
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD10]], [[VEC_IND]]
+; NORMAL_DEP_LIMIT-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP14]], align 4, !alias.scope [[META7]], !noalias [[META0]]
+; NORMAL_DEP_LIMIT-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
+; NORMAL_DEP_LIMIT-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; NORMAL_DEP_LIMIT-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; NORMAL_DEP_LIMIT:       middle.block:
+; NORMAL_DEP_LIMIT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; NORMAL_DEP_LIMIT-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; NORMAL_DEP_LIMIT:       scalar.ph:
+; NORMAL_DEP_LIMIT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NORMAL_DEP_LIMIT-NEXT:    br label [[FOR_BODY1:%.*]]
+; NORMAL_DEP_LIMIT:       for.body:
+; NORMAL_DEP_LIMIT-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; NORMAL_DEP_LIMIT-NEXT:    [[GEP_INDICES1:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES1]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; NORMAL_DEP_LIMIT-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; NORMAL_DEP_LIMIT-NEXT:    store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
+; NORMAL_DEP_LIMIT-NEXT:    store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[GEP_OTHER:%.*]] = getelementptr inbounds i32, ptr [[OTHER]], i64 [[IV1]]
+; NORMAL_DEP_LIMIT-NEXT:    [[L_OTHER:%.*]] = load i32, ptr [[GEP_OTHER]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[ADD_OTHER:%.*]] = add i32 [[L_OTHER]], [[IV_TRUNC]]
+; NORMAL_DEP_LIMIT-NEXT:    store i32 [[ADD_OTHER]], ptr [[GEP_OTHER]], align 4
+; NORMAL_DEP_LIMIT-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; NORMAL_DEP_LIMIT-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; NORMAL_DEP_LIMIT-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
+; NORMAL_DEP_LIMIT:       for.exit:
+; NORMAL_DEP_LIMIT-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %idx.addr = getelementptr inbounds i32, ptr %array, i64 %iv
+  %iv.trunc = trunc i64 %iv to i32
+  store i32 %iv.trunc, ptr %idx.addr, align 4
+  %gep.other = getelementptr inbounds i32, ptr %other, i64 %iv
+  %l.other = load i32, ptr %gep.other, align 4
+  %add.other = add i32 %l.other, %iv.trunc
+  store i32 %add.other, ptr %gep.other, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
new file mode 100644
index 0000000000000..9be068ce880ea
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -force-vector-interleave=1 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Based on the following C code:
+;;
+;; void simple_histogram(int *buckets, unsigned *indices, int N) {
+;;   for (int i = 0; i < N; ++i)
+;;     buckets[indices[i]]++;
+;; }
+
+;; Check that the scalar plan contains the original instructions.
+; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' {
+; CHECK-NEXT: Live-in [[VFxUF:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT [[IV:.*]] = CANONICAL-INDUCTION ir<0>, [[IV_NEXT:.*]]
+; CHECK-NEXT:     [[STEPS:vp.*]] = SCALAR-STEPS [[IV]], ir<1>
+; CHECK-NEXT:     CLONE [[GEP_IDX:.*]] = getelementptr inbounds ir<%indices>, [[STEPS]]
+; CHECK-NEXT:     CLONE [[IDX:.*]] = load [[GEP_IDX]]
+; CHECK-NEXT:     CLONE [[EXT_IDX:.*]] = zext [[IDX]]
+; CHECK-NEXT:     CLONE [[GEP_BUCKET:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX]]
+; CHECK-NEXT:     CLONE [[HISTVAL:.*]] = load [[GEP_BUCKET]]
+; CHECK-NEXT:     CLONE [[UPDATE:.*]] = add nsw [[HISTVAL]], ir<1>
+; CHECK-NEXT:     CLONE store [[UPDATE]], [[GEP_BUCKET]]
+; CHECK-NEXT:     EMIT [[IV_NEXT]] = add nuw [[IV]], [[VFxUF]]
+; CHECK-NEXT:     EMIT branch-on-count [[IV_NEXT]], [[VTC]]
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT:   EMIT [[TC_CHECK:.*]] = icmp eq [[OTC:.*]], [[VTC]]
+; CHECK-NEXT:   EMIT branch-on-cond [[TC_CHECK]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+;; Check that the vectorized plan contains a histogram recipe instead.
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF>=1' {
+; CHECK-NEXT: Live-in [[VFxUF:.*]] = VF * UF
+; CHECK-NEXT: Live-in [[VTC:.*]] = vector-trip-count
+; CHECK-NEXT: Live-in [[OTC:.*]] = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT [[IV:.*]] = CANONICAL-INDUCTION ir<0>, [[IV_NEXT:.*]]
+; CHECK-NEXT:     [[STEPS:vp.*]] = SCALAR-STEPS [[IV]], ir<1>
+; CHECK-NEXT:     CLONE [[GEP_IDX:.*]] = getelementptr inbounds ir<%indices>, [[STEPS]]
+; CHECK-NEXT:     [[VECP_IDX:vp.*]] = vector-pointer [[GEP_IDX]]
+; CHECK-NEXT:     WIDEN [[IDX:.*]] = load [[VECP_IDX]]
+; CHECK-NEXT:     WIDEN-CAST [[EXT_IDX:.*]] = zext  [[IDX]] to i64
+; CHECK-NEXT:     WIDEN-GEP Inv[Var] [[GEP_BUCKET:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX]]
+; CHECK-NEXT:     WIDEN-HISTOGRAM buckets: [[GEP_BUCKET]], inc: ir<1>
+; CHECK-NEXT:     EMIT [[IV_NEXT]] = add nuw [[IV]], [[VFxUF]]
+; CHECK-NEXT:     EMIT branch-on-count [[IV_NEXT]], [[VTC]]
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT:   EMIT [[TC_CHECK:.*]] = icmp eq [[OTC]], [[VTC]]
+; CHECK-NEXT:   EMIT branch-on-cond [[TC_CHECK]]
+; CHECK-NEXT: Successor(s): ir-bb<for.exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<for.exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
new file mode 100644
index 0000000000000..0c41477f285d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -0,0 +1,937 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; Based on the following C code:
+;;
+;; void simple_histogram(int *buckets, unsigned *indices, int N) {
+;;   for (int i = 0; i < N; ++i)
+;;     buckets[indices[i]]++;
+;; }
+
+;; Confirm finding a histogram operation
+; CHECK-LABEL: Checking a loop in 'simple_histogram'
+; CHECK: LV: Checking for a histogram on: store i32 %inc, ptr %gep.bucket, align 4
+; CHECK: LV: Found histogram for: store i32 %inc, ptr %gep.bucket, align 4
+
+;; Confirm cost calculation for runtime checks
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_rtdepcheck'
+; CHECK: Calculating cost of runtime checks:
+; CHECK: Total cost of runtime checks:
+; CHECK: LV: Minimum required TC for runtime checks to be profitable:
+
+;; Confirm inability to vectorize with potential alias to buckets
+; CHECK-LABEL: LV: Checking a loop in 'simple_histogram_unsafe_alias'
+; CHECK: LV: Can't vectorize due to memory conflicts
+; CHECK-NEXT: LV: Not vectorizing: Cannot prove legality.
+
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indices, i64 %N, i32 %incval) #0 {
+; CHECK-LABEL: define void @simple_histogram_inc_param(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]], i32 [[INCVAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 [[INCVAL]], <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], [[INCVAL]]
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, %incval
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_sub(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 -1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], -1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = sext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = sub nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices, ptr readonly %conds, i64 %N) #0 {
+; CHECK-LABEL: define void @conditional_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[CONDS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD1]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 5100, i64 0), <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[NEXT:%.*]] ]
+; CHECK-NEXT:    [[CONDIDX:%.*]] = getelementptr inbounds i32, ptr [[CONDS]], i64 [[IV1]]
+; CHECK-NEXT:    [[CONDDATA:%.*]] = load i32, ptr [[CONDIDX]], align 4
+; CHECK-NEXT:    [[IFCOND:%.*]] = icmp sgt i32 [[CONDDATA]], 5100
+; CHECK-NEXT:    br i1 [[IFCOND]], label [[IFTRUE:%.*]], label [[NEXT]]
+; CHECK:       iftrue:
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    br label [[NEXT]]
+; CHECK:       next:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %next ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %condidx = getelementptr inbounds i32, ptr %conds, i64 %iv
+  %conddata = load i32, ptr %condidx, align 4
+  %ifcond = icmp sgt i32 %conddata, 5100
+  br i1 %ifcond, label %iftrue, label %next
+
+iftrue:
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  br label %next
+
+next:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_8bit(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw nsw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP6]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i8(<vscale x 4 x ptr> [[TMP7]], i8 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i8 [[TMP1]], 1
+; CHECK-NEXT:    store i8 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i8, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i8, ptr %gep.bucket, align 4
+  %inc = add nsw i8 %l.bucket, 1
+  store i8 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+;; We don't currently support floating point histograms.
+define void @histogram_float(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_float(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
+; CHECK-NEXT:    store float [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds float, ptr %buckets, i64 %idxprom1
+  %l.bucket = load float, ptr %gep.bucket, align 4
+  %inc = fadd fast float %l.bucket, 1.0
+  store float %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+;; We don't support histograms with a update value that's not loop-invariant.
+define void @histogram_varying_increment(ptr noalias %buckets, ptr readonly %indices, ptr readonly %incvals, i64 %N) #0 {
+; CHECK-LABEL: define void @histogram_varying_increment(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly [[INCVALS:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INCIDX:%.*]] = getelementptr inbounds i32, ptr [[INCVALS]], i64 [[IV]]
+; CHECK-NEXT:    [[INCVAL:%.*]] = load i32, ptr [[INCIDX]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP1]], [[INCVAL]]
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP12]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %gep.incvals = getelementptr inbounds i32, ptr %incvals, i64 %iv
+  %l.incval = load i32, ptr %gep.incvals, align 4
+  %inc = add nsw i32 %l.bucket, %l.incval
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+;; Test that interleaving works when vectorizing.
+define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_user_interleave(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTIDX:%.*]] = shl nuw nsw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[DOTIDX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP17]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP19:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP19]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP21]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !0
+
+for.exit:
+  ret void
+}
+
+;; Test that we can handle more than one GEP index.
+@idx_array = dso_local local_unnamed_addr global [1048576 x i32] zeroinitializer, align 4
+@data_array = dso_local local_unnamed_addr global [1048576 x i32] zeroinitializer, align 4
+
+define void @histogram_array_3op_gep(i64 noundef %N) #0 {
+; CHECK-LABEL: define void @histogram_array_3op_gep(
+; CHECK-SAME: i64 noundef [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, <vscale x 4 x i64> [[TMP14]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP11]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 [[IV]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 [[IDXPROM5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds [1048576 x i32], ptr @idx_array, i64 0, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom5 = sext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds [1048576 x i32], ptr @data_array, i64 0, i64 %idxprom5
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+;; Add a struct into the mix, use a different constant index.
+;; { unused, buckets }
+%somestruct = type { [1048576 x i32], [1048576 x i32] }
+
+define void @histogram_array_4op_gep_nonzero_const_idx(i64 noundef %N, ptr readonly %indices, ptr noalias %data.struct) #0 {
+; CHECK-LABEL: define void @histogram_array_4op_gep_nonzero_const_idx(
+; CHECK-SAME: i64 noundef [[N:%.*]], ptr readonly [[INDICES:%.*]], ptr noalias [[DATA_STRUCT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[SOMESTRUCT:%.*]], ptr [[DATA_STRUCT]], i64 1, i32 0, <vscale x 4 x i64> [[TMP6]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP7]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
+; CHECK-NEXT:    [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[L_IDX]] to i64
+; CHECK-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds [[SOMESTRUCT]], ptr [[DATA_STRUCT]], i64 1, i32 0, i64 [[IDXPROM5]]
+; CHECK-NEXT:    [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom5 = sext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds %somestruct, ptr %data.struct, i32 1, i32 0, i64 %idxprom5
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+;; Make sure the histogram intrinsic uses the active lane mask when tail folding.
+define void @simple_histogram_tailfold(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_tailfold(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP5]])
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP9]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP10]], i32 1, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT:    br i1 [[TMP11]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 poison, label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !2
+
+for.exit:
+  ret void
+}
+
+;; Check that we can still vectorize a histogram when LAA found another dependency
+;; that doesn't conflict with the buckets.
+define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_rtdepcheck(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[ARRAY1:%.*]] = ptrtoint ptr [[ARRAY]] to i64
+; CHECK-NEXT:    [[INDICES2:%.*]] = ptrtoint ptr [[INDICES]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[ARRAY1]], [[INDICES2]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP6]], -4
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP11:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP13]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP14]], i32 1, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT:    store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP15]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP18]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IDX_ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    store i32 [[IV_TRUNC]], ptr [[IDX_ADDR]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %idx.addr = getelementptr inbounds i32, ptr %array, i64 %iv
+  %iv.trunc = trunc i64 %iv to i32
+  store i32 %iv.trunc, ptr %idx.addr, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
+
+;; Make sure we don't vectorize if there's a potential alias between buckets
+;; and indices.
+define void @simple_histogram_unsafe_alias(ptr %buckets, ptr %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_unsafe_alias(
+; CHECK-SAME: ptr [[BUCKETS:%.*]], ptr [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
+
+define void @simple_histogram_64b(ptr noalias %buckets, ptr readonly %indices, i64 %N) #0 {
+; CHECK-LABEL: define void @simple_histogram_64b(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -2
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[INDICES]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[BUCKETS]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> [[TMP6]], i64 1, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[GEP_INDICES:%.*]] = getelementptr inbounds i64, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[L_IDX:%.*]] = load i64, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT:    [[GEP_BUCKET:%.*]] = getelementptr inbounds i64, ptr [[BUCKETS]], i64 [[L_IDX]]
+; CHECK-NEXT:    [[L_BUCKET:%.*]] = load i64, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i64 [[L_BUCKET]], 1
+; CHECK-NEXT:    store i64 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i64, ptr %indices, i64 %iv
+  %l.idx = load i64, ptr %gep.indices, align 4
+  %gep.bucket = getelementptr inbounds i64, ptr %buckets, i64 %l.idx
+  %l.bucket = load i64, ptr %gep.bucket, align 4
+  %inc = add nsw i64 %l.bucket, 1
+  store i64 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body, !llvm.loop !4
+
+for.exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+sve2" vscale_range(1,16) }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.interleave.count", i32 1}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
new file mode 100644
index 0000000000000..870925950ae49
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s --prefer-predicate-over-epilogue=predicate-dont-vectorize --passes=loop-vectorize -mcpu=sifive-p470 -mattr=+v,+f -force-tail-folding-style=data-with-evl -S | FileCheck %s
+; Generated from issue #109468.
+; In this test case, the vector store with tail mask will transfer to the vp intrinsic with EVL.
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
+; CHECK-LABEL: define void @lshift_significand(
+; CHECK-SAME: i32 [[N:%.*]], ptr nocapture writeonly [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP1_PEEL]], i64 2, i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 3, [[SPEC_SELECT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[SPEC_SELECT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP0]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP10]], i32 2, i1 true)
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[SPEC_SELECT]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = sub nuw nsw i64 1, [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 0, [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sub i64 1, [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[TMP14]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr [[TMP19]], i64 [[TMP18]]
+; CHECK-NEXT:    [[VP_REVERSE:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.reverse.nxv2i64(<vscale x 2 x i64> zeroinitializer, <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP11]])
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_REVERSE]], ptr align 8 [[TMP20]], <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[SPEC_SELECT]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = sub nuw nsw i64 1, [[IV]]
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP23]]
+; CHECK-NEXT:    store i64 0, ptr [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 3
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1.peel = icmp eq i32 %n, 0
+  %spec.select = select i1 %cmp1.peel, i64 2, i64 0
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %spec.select, %entry ], [ %iv.next, %loop ]
+  %1 = sub nuw nsw i64 1, %iv
+  %arrayidx13 = getelementptr i64, ptr %dst, i64 %1
+  store i64 0, ptr %arrayidx13, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 3
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll b/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
index 40c143faadd3c..d956f718cb10a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/iv-live-outs.ll
@@ -17,13 +17,7 @@ define i64 @test_pr98660(ptr %dst, i64 %N) {
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 24
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP3]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP9]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[TMP9]], i32 8
diff --git a/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
index a768f69f49788..12f95e0a0a7dc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
@@ -35,15 +35,9 @@ define void @iv.4_used_as_vector_and_first_lane(ptr %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD4]], <i64 128, i64 128, i64 128, i64 128>
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD5]], <i64 128, i64 128, i64 128, i64 128>
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD6]], <i64 128, i64 128, i64 128, i64 128>
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i64> [[TMP12]], i32 0
-; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP20]], 1
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i64> [[TMP13]], i32 0
-; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP22]], 1
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i64> [[TMP14]], i32 0
-; CHECK-NEXT:    [[TMP25:%.*]] = add i64 [[TMP24]], 1
-; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i64> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i64> [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP27:%.*]] = add i64 [[TMP26]], 1
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP27]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i64, ptr [[TMP28]], i32 0
 ; CHECK-NEXT:    [[TMP33:%.*]] = getelementptr i64, ptr [[TMP28]], i32 4
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[TMP28]], i32 8
@@ -116,9 +110,6 @@ define void @iv.4_used_as_first_lane(ptr %src, ptr noalias %dst) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 4
@@ -128,19 +119,13 @@ define void @iv.4_used_as_first_lane(ptr %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP10]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP1]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP2]], 4
-; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD]], <i64 128, i64 128, i64 128, i64 128>
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD1]], <i64 128, i64 128, i64 128, i64 128>
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD2]], <i64 128, i64 128, i64 128, i64 128>
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp ule <4 x i64> [[WIDE_LOAD3]], <i64 128, i64 128, i64 128, i64 128>
-; CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[TMP12]], 1
-; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP13]], 1
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[TMP14]], 1
 ; CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[TMP15]], 1
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP23]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i64, ptr [[TMP24]], i32 0
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr i64, ptr [[TMP24]], i32 4
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[TMP24]], i32 8
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
new file mode 100644
index 0000000000000..5f0e3b1e10d3e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -force-vector-width=2 -S | FileCheck %s
+
+;; Currently we don't expect this to vectorize, since the generic cost model returns
+;; invalid for the histogram intrinsic.
+define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
+; CHECK-LABEL: define void @simple_histogram(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
+; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+  %l.idx = load i32, ptr %gep.indices, align 4
+  %idxprom1 = zext i32 %l.idx to i64
+  %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+  %l.bucket = load i32, ptr %gep.bucket, align 4
+  %inc = add nsw i32 %l.bucket, 1
+  store i32 %inc, ptr %gep.bucket, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %N
+  br i1 %exitcond, label %for.exit, label %for.body
+
+for.exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index ba260752ce4b5..0114b9c244358 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -812,6 +812,94 @@ define void @memcpy_memory_none(ptr %p, ptr %p2, i64 %size) {
   ret void
 }
 
+declare void @do_something()
+declare void @capture(ptr)
+
+define void @memcpy_memcpy_escape_before(ptr noalias %P, ptr noalias %Q) {
+; CHECK-LABEL: @memcpy_memcpy_escape_before(
+; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [32 x i8], align 16
+; CHECK-NEXT:    call void @capture(ptr [[MEMTMP]])
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[MEMTMP]], ptr align 16 [[P:%.*]], i32 32, i1 false)
+; CHECK-NEXT:    call void @do_something()
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[Q:%.*]], ptr align 16 [[MEMTMP]], i32 32, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %memtmp = alloca [32 x i8], align 16
+  call void @capture(ptr %memtmp)
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %memtmp, ptr align 16 %P, i32 32, i1 false)
+  call void @do_something()
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %Q, ptr align 16 %memtmp, i32 32, i1 false)
+  ret void
+}
+
+define void @memcpy_memcpy_escape_after1(ptr noalias %P, ptr noalias %Q) {
+; CHECK-LABEL: @memcpy_memcpy_escape_after1(
+; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [32 x i8], align 16
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[MEMTMP]], ptr align 16 [[P:%.*]], i32 32, i1 false)
+; CHECK-NEXT:    call void @do_something()
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[Q:%.*]], ptr align 16 [[MEMTMP]], i32 32, i1 false)
+; CHECK-NEXT:    call void @capture(ptr [[MEMTMP]])
+; CHECK-NEXT:    ret void
+;
+  %memtmp = alloca [32 x i8], align 16
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %memtmp, ptr align 16 %P, i32 32, i1 false)
+  call void @do_something()
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %Q, ptr align 16 %memtmp, i32 32, i1 false)
+  call void @capture(ptr %memtmp)
+  ret void
+}
+
+define void @memcpy_memcpy_escape_after2(ptr noalias %P, ptr noalias %Q) {
+; CHECK-LABEL: @memcpy_memcpy_escape_after2(
+; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [32 x i8], align 16
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[MEMTMP]], ptr align 16 [[P:%.*]], i32 32, i1 false)
+; CHECK-NEXT:    call void @do_something()
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 16 [[Q:%.*]], ptr align 16 [[MEMTMP]], i32 32, i1 false)
+; CHECK-NEXT:    call void @capture(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %memtmp = alloca [32 x i8], align 16
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %memtmp, ptr align 16 %P, i32 32, i1 false)
+  call void @do_something()
+  call void @llvm.memcpy.p0.p0.i32(ptr align 16 %Q, ptr align 16 %memtmp, i32 32, i1 false)
+  call void @capture(ptr %P)
+  ret void
+}
+
+define void @memcpy_byval_escape_after(ptr noalias %P) {
+; CHECK-LABEL: @memcpy_byval_escape_after(
+; CHECK-NEXT:    [[A:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[A]], ptr align 4 [[P:%.*]], i64 8, i1 false)
+; CHECK-NEXT:    call void @do_something()
+; CHECK-NEXT:    call void @test4a(ptr byval(i8) align 1 [[A]])
+; CHECK-NEXT:    call void @capture(ptr [[P]])
+; CHECK-NEXT:    ret void
+;
+  %A = alloca [8 x i8]
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %A, ptr align 4 %P, i64 8, i1 false)
+  call void @do_something()
+  call void @test4a(ptr align 1 byval(i8) %A)
+  call void @capture(ptr %P)
+  ret void
+}
+
+define void @memcpy_immut_escape_after(ptr align 4 noalias %val) {
+; CHECK-LABEL: @memcpy_immut_escape_after(
+; CHECK-NEXT:    [[VAL1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[VAL1]], ptr align 4 [[VAL:%.*]], i64 1, i1 false)
+; CHECK-NEXT:    call void @do_something()
+; CHECK-NEXT:    call void @f(ptr noalias nocapture readonly align 4 [[VAL1]])
+; CHECK-NEXT:    call void @capture(ptr [[VAL]])
+; CHECK-NEXT:    ret void
+;
+  %val1 = alloca i8, align 4
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %val1, ptr align 4 %val, i64 1, i1 false)
+  call void @do_something()
+  call void @f(ptr align 4 nocapture noalias readonly %val1)
+  call void @capture(ptr %val)
+  ret void
+}
+
 !0 = !{!0}
 !1 = !{!1, !0}
 !2 = !{!1}
diff --git a/llvm/test/Transforms/MemProfContextDisambiguation/inlined4.ll b/llvm/test/Transforms/MemProfContextDisambiguation/inlined4.ll
new file mode 100644
index 0000000000000..bf419ea987bd0
--- /dev/null
+++ b/llvm/test/Transforms/MemProfContextDisambiguation/inlined4.ll
@@ -0,0 +1,102 @@
+;; This test ensures that the logic which assigns calls to stack nodes
+;; correctly handles a case where multiple nodes have stack ids that
+;; overlap with each other but have different last nodes (can happen with
+;; inlining into various levels of a call chain). Specifically, when we
+;; have one that is duplicated (e.g. unrolling), we need to correctly
+;; handle the case where the context id has already been assigned to
+;; a different call with a different last node.
+
+;; -stats requires asserts
+; REQUIRES: asserts
+
+; RUN: opt -passes=memprof-context-disambiguation -supports-hot-cold-new \
+; RUN:	-memprof-verify-ccg -memprof-verify-nodes \
+; RUN:  -stats -pass-remarks=memprof-context-disambiguation \
+; RUN:	%s -S 2>&1 | FileCheck %s --check-prefix=IR \
+; RUN:  --check-prefix=STATS --check-prefix=REMARKS
+
+; REMARKS: created clone _Z1Ab.memprof.1
+; REMARKS: created clone _Z3XZNv.memprof.1
+; REMARKS: call in clone main assigned to call function clone _Z3XZNv.memprof.1
+;; Make sure the inlined context in _Z3XZNv, which partially overlaps the stack
+;; ids in the shorter inlined context of Z2XZv, correctly calls a cloned
+;; version of Z1Ab, which will call the cold annotated allocation.
+; REMARKS: call in clone _Z3XZNv.memprof.1 assigned to call function clone _Z1Ab.memprof.1
+; REMARKS: call in clone _Z1Ab.memprof.1 marked with memprof allocation attribute cold
+; REMARKS: call in clone main assigned to call function clone _Z3XZNv
+; REMARKS: call in clone _Z3XZNv assigned to call function clone _Z1Ab
+; REMARKS: call in clone _Z1Ab marked with memprof allocation attribute notcold
+
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @_Z1Ab() {
+entry:
+  %call = call noalias noundef nonnull ptr @_Znam(i64 noundef 10) #1, !memprof !0, !callsite !5
+  ret void
+}
+
+; Function Attrs: nobuiltin
+declare ptr @_Znam(i64) #0
+
+;; Inlining of stack id 2 into 3. Assume this is called from somewhere else.
+define dso_local void @_Z2XZv() local_unnamed_addr #0 {
+entry:
+  ;; Simulate duplication of the callsite (e.g. unrolling).
+  call void @_Z1Ab(), !callsite !6
+  call void @_Z1Ab(), !callsite !6
+  ret void
+}
+
+;; Inlining of stack id 2 into 3 into 4. Called by main below.
+define dso_local void @_Z3XZNv() local_unnamed_addr {
+entry:
+  call void @_Z1Ab(), !callsite !7
+  ret void
+}
+
+define dso_local noundef i32 @main() local_unnamed_addr {
+entry:
+  call void @_Z3XZNv(), !callsite !8 ;; Not cold context
+  call void @_Z3XZNv(), !callsite !9 ;; Cold context
+  ret i32 0
+}
+
+attributes #0 = { nobuiltin }
+attributes #7 = { builtin }
+
+!0 = !{!1, !3}
+;; Not cold context via first call to _Z3XZNv in main
+!1 = !{!2, !"notcold"}
+!2 = !{i64 1, i64 2, i64 3, i64 4, i64 5}
+;; Cold context via second call to _Z3XZNv in main
+!3 = !{!4, !"cold"}
+!4 = !{i64 1, i64 2, i64 3, i64 4, i64 6}
+!5 = !{i64 1}
+!6 = !{i64 2, i64 3}
+!7 = !{i64 2, i64 3, i64 4}
+!8 = !{i64 5}
+!9 = !{i64 6}
+
+; IR: define {{.*}} @_Z1Ab()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[NOTCOLD:[0-9]+]]
+; IR: define {{.*}} @_Z2XZv()
+; IR:   call {{.*}} @_Z1Ab()
+; IR:   call {{.*}} @_Z1Ab()
+; IR: define {{.*}} @_Z3XZNv()
+; IR:   call {{.*}} @_Z1Ab()
+; IR: define {{.*}} @main()
+; IR:   call {{.*}} @_Z3XZNv()
+; IR:   call {{.*}} @_Z3XZNv.memprof.1()
+; IR: define {{.*}} @_Z1Ab.memprof.1()
+; IR:   call {{.*}} @_Znam(i64 noundef 10) #[[COLD:[0-9]+]]
+; IR: define {{.*}} @_Z3XZNv.memprof.1()
+; IR:   call {{.*}} @_Z1Ab.memprof.1()
+
+; IR: attributes #[[NOTCOLD]] = { "memprof"="notcold" }
+; IR: attributes #[[COLD]] = { "memprof"="cold" }
+
+; STATS: 1 memprof-context-disambiguation - Number of cold static allocations (possibly cloned)
+; STATS: 1 memprof-context-disambiguation - Number of not cold static allocations (possibly cloned)
+; STATS: 2 memprof-context-disambiguation - Number of function clones created during whole program analysis
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/minbitwidth-cast-node-length.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/minbitwidth-cast-node-length.ll
new file mode 100644
index 0000000000000..877c06ab7bcde
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/minbitwidth-cast-node-length.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=arm64-apple-macosx11.0.0 < %s | FileCheck %s
+
+define void @test(ptr %pDst, i32 %stride, i8 %0, ptr %p1, ptr %p2, ptr %p4, ptr %p3) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[PDST:%.*]], i32 [[STRIDE:%.*]], i8 [[TMP0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P4:%.*]], ptr [[P3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[MUL100:%.*]] = mul i32 [[STRIDE]], 9
+; CHECK-NEXT:    [[MUL101:%.*]] = mul i32 [[STRIDE]], 7
+; CHECK-NEXT:    [[MUL102:%.*]] = mul i32 [[STRIDE]], 5
+; CHECK-NEXT:    [[MUL103:%.*]] = mul i32 [[STRIDE]], 3
+; CHECK-NEXT:    [[CONV111:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL112:%.*]] = mul i32 [[CONV111]], 14
+; CHECK-NEXT:    [[CONV117:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL118:%.*]] = mul i32 [[CONV117]], 14
+; CHECK-NEXT:    [[CONV124:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL125:%.*]] = mul i32 [[CONV124]], 14
+; CHECK-NEXT:    [[CONV131:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL132:%.*]] = mul i32 [[CONV131]], 14
+; CHECK-NEXT:    [[CMP139:%.*]] = icmp uge i32 [[MUL112]], [[MUL100]]
+; CHECK-NEXT:    [[CMP142:%.*]] = icmp uge i32 [[MUL112]], [[MUL101]]
+; CHECK-NEXT:    [[ADD1441:%.*]] = or i1 [[CMP139]], [[CMP142]]
+; CHECK-NEXT:    [[CMP145:%.*]] = icmp uge i32 [[MUL112]], [[MUL102]]
+; CHECK-NEXT:    [[ADD1472:%.*]] = or i1 [[ADD1441]], [[CMP145]]
+; CHECK-NEXT:    [[CMP148:%.*]] = icmp uge i32 [[MUL112]], [[MUL103]]
+; CHECK-NEXT:    [[ADD1504:%.*]] = or i1 [[ADD1472]], [[CMP148]]
+; CHECK-NEXT:    [[ADD151:%.*]] = zext i1 [[ADD1504]] to i64
+; CHECK-NEXT:    [[ARRAYIDX156:%.*]] = getelementptr [8 x i32], ptr [[P1]], i64 0, i64 [[ADD151]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX156]], align 4
+; CHECK-NEXT:    [[CMP165:%.*]] = icmp uge i32 [[MUL118]], [[MUL101]]
+; CHECK-NEXT:    [[CMP171:%.*]] = icmp uge i32 [[MUL118]], [[MUL103]]
+; CHECK-NEXT:    [[ADD1734:%.*]] = or i1 [[CMP165]], [[CMP171]]
+; CHECK-NEXT:    [[ADD173:%.*]] = zext i1 [[ADD1734]] to i64
+; CHECK-NEXT:    [[ARRAYIDX178:%.*]] = getelementptr [8 x i32], ptr [[P2]], i64 0, i64 [[ADD173]]
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX178]], align 4
+; CHECK-NEXT:    [[CMP185:%.*]] = icmp uge i32 [[MUL125]], [[MUL100]]
+; CHECK-NEXT:    [[CMP188:%.*]] = icmp uge i32 [[MUL125]], [[MUL101]]
+; CHECK-NEXT:    [[ADD1905:%.*]] = or i1 [[CMP185]], [[CMP188]]
+; CHECK-NEXT:    [[CMP191:%.*]] = icmp uge i32 [[MUL125]], [[MUL102]]
+; CHECK-NEXT:    [[ADD1936:%.*]] = or i1 [[ADD1905]], [[CMP191]]
+; CHECK-NEXT:    [[ADD193:%.*]] = zext i1 [[ADD1936]] to i64
+; CHECK-NEXT:    [[ARRAYIDX201:%.*]] = getelementptr [8 x i32], ptr [[P4]], i64 0, i64 [[ADD193]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX201]], align 4
+; CHECK-NEXT:    [[CMP208:%.*]] = icmp uge i32 [[MUL132]], [[MUL100]]
+; CHECK-NEXT:    [[CMP211:%.*]] = icmp uge i32 [[MUL132]], [[MUL101]]
+; CHECK-NEXT:    [[ADD2137:%.*]] = or i1 [[CMP208]], [[CMP211]]
+; CHECK-NEXT:    [[CMP214:%.*]] = icmp uge i32 [[MUL132]], [[MUL102]]
+; CHECK-NEXT:    [[ADD2168:%.*]] = or i1 [[ADD2137]], [[CMP214]]
+; CHECK-NEXT:    [[CMP217:%.*]] = icmp uge i32 [[MUL132]], [[MUL103]]
+; CHECK-NEXT:    [[ADD2199:%.*]] = or i1 [[ADD2168]], [[CMP217]]
+; CHECK-NEXT:    [[ADD219:%.*]] = zext i1 [[ADD2199]] to i64
+; CHECK-NEXT:    [[ARRAYIDX224:%.*]] = getelementptr [8 x i32], ptr [[P3]], i64 0, i64 [[ADD219]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX224]], align 4
+; CHECK-NEXT:    [[CONV230:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL231:%.*]] = mul i32 [[CONV230]], 14
+; CHECK-NEXT:    [[CONV237:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL238:%.*]] = mul i32 [[CONV237]], 14
+; CHECK-NEXT:    [[CONV244:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL245:%.*]] = mul i32 [[CONV244]], 14
+; CHECK-NEXT:    [[CONV484:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL485:%.*]] = mul i32 [[CONV484]], 14
+; CHECK-NEXT:    [[CMP262:%.*]] = icmp uge i32 [[MUL231]], [[MUL101]]
+; CHECK-NEXT:    [[CMP268:%.*]] = icmp uge i32 [[MUL231]], [[MUL103]]
+; CHECK-NEXT:    [[ADD1503:%.*]] = or i1 [[CMP262]], [[CMP268]]
+; CHECK-NEXT:    [[ADD150:%.*]] = zext i1 [[ADD1503]] to i64
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr [8 x i32], ptr [[P1]], i64 0, i64 [[ADD150]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX155]], align 4
+; CHECK-NEXT:    [[OR951:%.*]] = or i32 [[TMP13]], [[TMP18]]
+; CHECK-NEXT:    [[CMP282:%.*]] = icmp uge i32 [[MUL238]], [[MUL100]]
+; CHECK-NEXT:    [[CMP285:%.*]] = icmp uge i32 [[MUL238]], [[MUL101]]
+; CHECK-NEXT:    [[ADD28711:%.*]] = or i1 [[CMP282]], [[CMP285]]
+; CHECK-NEXT:    [[CMP288:%.*]] = icmp uge i32 [[MUL238]], [[MUL102]]
+; CHECK-NEXT:    [[ADD29012:%.*]] = or i1 [[ADD28711]], [[CMP288]]
+; CHECK-NEXT:    [[CMP291:%.*]] = icmp uge i32 [[MUL238]], [[MUL103]]
+; CHECK-NEXT:    [[ADD29313:%.*]] = or i1 [[ADD29012]], [[CMP291]]
+; CHECK-NEXT:    [[ADD293:%.*]] = zext i1 [[ADD29313]] to i64
+; CHECK-NEXT:    [[ARRAYIDX298:%.*]] = getelementptr [8 x i32], ptr [[P2]], i64 0, i64 [[ADD293]]
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX298]], align 4
+; CHECK-NEXT:    [[OR301952:%.*]] = or i32 [[TMP21]], [[TMP12]]
+; CHECK-NEXT:    [[CMP310:%.*]] = icmp uge i32 [[MUL245]], [[MUL101]]
+; CHECK-NEXT:    [[CMP316:%.*]] = icmp uge i32 [[MUL245]], [[MUL103]]
+; CHECK-NEXT:    [[ADD31814:%.*]] = or i1 [[CMP310]], [[CMP316]]
+; CHECK-NEXT:    [[ADD318:%.*]] = zext i1 [[ADD31814]] to i64
+; CHECK-NEXT:    [[ARRAYIDX323:%.*]] = getelementptr [8 x i32], ptr [[P4]], i64 0, i64 [[ADD318]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX323]], align 4
+; CHECK-NEXT:    [[OR326953:%.*]] = or i32 [[TMP14]], [[TMP19]]
+; CHECK-NEXT:    [[CMP332:%.*]] = icmp uge i32 [[MUL485]], [[MUL100]]
+; CHECK-NEXT:    [[CMP335:%.*]] = icmp uge i32 [[MUL485]], [[MUL101]]
+; CHECK-NEXT:    [[ADD33715:%.*]] = or i1 [[CMP332]], [[CMP335]]
+; CHECK-NEXT:    [[CMP338:%.*]] = icmp uge i32 [[MUL485]], [[MUL102]]
+; CHECK-NEXT:    [[ADD34016:%.*]] = or i1 [[ADD33715]], [[CMP338]]
+; CHECK-NEXT:    [[CMP341:%.*]] = icmp uge i32 [[MUL485]], [[MUL103]]
+; CHECK-NEXT:    [[ADD34317:%.*]] = or i1 [[ADD34016]], [[CMP341]]
+; CHECK-NEXT:    [[ADD343:%.*]] = zext i1 [[ADD34317]] to i64
+; CHECK-NEXT:    [[ARRAYIDX348:%.*]] = getelementptr [8 x i32], ptr [[P3]], i64 0, i64 [[ADD343]]
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX348]], align 4
+; CHECK-NEXT:    [[OR351954:%.*]] = or i32 [[TMP22]], [[TMP20]]
+; CHECK-NEXT:    [[CONV485:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL486:%.*]] = mul i32 [[CONV485]], 14
+; CHECK-NEXT:    [[CONV491:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL492:%.*]] = mul i32 [[CONV491]], 14
+; CHECK-NEXT:    [[CONV498:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL499:%.*]] = mul i32 [[CONV498]], 14
+; CHECK-NEXT:    [[CONV505:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT:    [[MUL506:%.*]] = mul i32 [[CONV505]], 14
+; CHECK-NEXT:    [[CMP519:%.*]] = icmp uge i32 [[MUL486]], [[MUL102]]
+; CHECK-NEXT:    [[CMP522:%.*]] = icmp uge i32 [[MUL486]], [[MUL103]]
+; CHECK-NEXT:    [[ADD52418:%.*]] = or i1 [[CMP519]], [[CMP522]]
+; CHECK-NEXT:    [[ADD524:%.*]] = zext i1 [[ADD52418]] to i64
+; CHECK-NEXT:    [[ARRAYIDX529:%.*]] = getelementptr [8 x i32], ptr [[P1]], i64 0, i64 [[ADD524]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX529]], align 4
+; CHECK-NEXT:    [[CMP541:%.*]] = icmp uge i32 [[MUL492]], [[MUL101]]
+; CHECK-NEXT:    [[CMP544:%.*]] = icmp uge i32 [[MUL492]], [[MUL102]]
+; CHECK-NEXT:    [[ADD54619:%.*]] = or i1 [[CMP541]], [[CMP544]]
+; CHECK-NEXT:    [[CMP547:%.*]] = icmp uge i32 [[MUL492]], [[MUL103]]
+; CHECK-NEXT:    [[ADD54920:%.*]] = or i1 [[ADD54619]], [[CMP547]]
+; CHECK-NEXT:    [[ADD549:%.*]] = zext i1 [[ADD54920]] to i64
+; CHECK-NEXT:    [[ARRAYIDX554:%.*]] = getelementptr [8 x i32], ptr [[P2]], i64 0, i64 [[ADD549]]
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[ARRAYIDX554]], align 4
+; CHECK-NEXT:    [[CMP572:%.*]] = icmp uge i32 [[MUL499]], [[MUL103]]
+; CHECK-NEXT:    [[CONV573:%.*]] = zext i1 [[CMP572]] to i64
+; CHECK-NEXT:    [[ARRAYIDX579:%.*]] = getelementptr [8 x i32], ptr [[P4]], i64 0, i64 [[CONV573]]
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX579]], align 4
+; CHECK-NEXT:    [[CMP594:%.*]] = icmp uge i32 [[MUL506]], [[MUL102]]
+; CHECK-NEXT:    [[CONV595:%.*]] = zext i1 [[CMP594]] to i64
+; CHECK-NEXT:    [[ARRAYIDX604:%.*]] = getelementptr [8 x i32], ptr [[P3]], i64 0, i64 [[CONV595]]
+; CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX604]], align 4
+; CHECK-NEXT:    [[OR4791159:%.*]] = or i32 [[OR301952]], [[OR951]]
+; CHECK-NEXT:    [[OR6071160:%.*]] = or i32 [[OR4791159]], [[OR326953]]
+; CHECK-NEXT:    [[OR4541161:%.*]] = or i32 [[OR6071160]], [[OR351954]]
+; CHECK-NEXT:    [[SHL58111621163:%.*]] = or i32 [[TMP27]], [[TMP29]]
+; CHECK-NEXT:    [[SHL55611641165:%.*]] = or i32 [[TMP25]], [[SHL58111621163]]
+; CHECK-NEXT:    [[SHL53111661167:%.*]] = or i32 [[TMP23]], [[SHL55611641165]]
+; CHECK-NEXT:    [[SHL5311166:%.*]] = trunc i32 [[SHL53111661167]] to i8
+; CHECK-NEXT:    [[CONV616:%.*]] = trunc i32 [[OR4541161]] to i8
+; CHECK-NEXT:    [[ARRAYIDX617:%.*]] = getelementptr i8, ptr [[PDST]], i64 4
+; CHECK-NEXT:    store i8 [[CONV616]], ptr [[ARRAYIDX617]], align 1
+; CHECK-NEXT:    store i8 [[SHL5311166]], ptr [[PDST]], align 1
+; CHECK-NEXT:    ret void
+;
+entry:
+  %mul100 = mul i32 %stride, 9
+  %mul101 = mul i32 %stride, 7
+  %mul102 = mul i32 %stride, 5
+  %mul103 = mul i32 %stride, 3
+  %conv111 = zext i8 %0 to i32
+  %mul112 = mul i32 %conv111, 14
+  %conv117 = zext i8 %0 to i32
+  %mul118 = mul i32 %conv117, 14
+  %conv124 = zext i8 %0 to i32
+  %mul125 = mul i32 %conv124, 14
+  %conv131 = zext i8 %0 to i32
+  %mul132 = mul i32 %conv131, 14
+  %cmp139 = icmp uge i32 %mul112, %mul100
+  %cmp142 = icmp uge i32 %mul112, %mul101
+  %add1441 = or i1 %cmp139, %cmp142
+  %cmp145 = icmp uge i32 %mul112, %mul102
+  %add1472 = or i1 %add1441, %cmp145
+  %cmp148 = icmp uge i32 %mul112, %mul103
+  %add1503 = or i1 %add1472, %cmp148
+  %add150 = zext i1 %add1503 to i64
+  %arrayidx155 = getelementptr [8 x i32], ptr %p1, i64 0, i64 %add150
+  %1 = load i32, ptr %arrayidx155, align 4
+  %cmp165 = icmp uge i32 %mul118, %mul101
+  %cmp171 = icmp uge i32 %mul118, %mul103
+  %add1734 = or i1 %cmp165, %cmp171
+  %add173 = zext i1 %add1734 to i64
+  %arrayidx178 = getelementptr [8 x i32], ptr %p2, i64 0, i64 %add173
+  %2 = load i32, ptr %arrayidx178, align 4
+  %cmp185 = icmp uge i32 %mul125, %mul100
+  %cmp188 = icmp uge i32 %mul125, %mul101
+  %add1905 = or i1 %cmp185, %cmp188
+  %cmp191 = icmp uge i32 %mul125, %mul102
+  %add1936 = or i1 %add1905, %cmp191
+  %add193 = zext i1 %add1936 to i64
+  %arrayidx201 = getelementptr [8 x i32], ptr %p4, i64 0, i64 %add193
+  %3 = load i32, ptr %arrayidx201, align 4
+  %cmp208 = icmp uge i32 %mul132, %mul100
+  %cmp211 = icmp uge i32 %mul132, %mul101
+  %add2137 = or i1 %cmp208, %cmp211
+  %cmp214 = icmp uge i32 %mul132, %mul102
+  %add2168 = or i1 %add2137, %cmp214
+  %cmp217 = icmp uge i32 %mul132, %mul103
+  %add2199 = or i1 %add2168, %cmp217
+  %add219 = zext i1 %add2199 to i64
+  %arrayidx224 = getelementptr [8 x i32], ptr %p3, i64 0, i64 %add219
+  %4 = load i32, ptr %arrayidx224, align 4
+  %conv230 = zext i8 %0 to i32
+  %mul231 = mul i32 %conv230, 14
+  %conv237 = zext i8 %0 to i32
+  %mul238 = mul i32 %conv237, 14
+  %conv244 = zext i8 %0 to i32
+  %mul245 = mul i32 %conv244, 14
+  %conv251 = zext i8 %0 to i32
+  %mul252 = mul i32 %conv251, 14
+  %cmp262 = icmp uge i32 %mul231, %mul101
+  %cmp268 = icmp uge i32 %mul231, %mul103
+  %add27010 = or i1 %cmp262, %cmp268
+  %add270 = zext i1 %add27010 to i64
+  %arrayidx275 = getelementptr [8 x i32], ptr %p1, i64 0, i64 %add270
+  %5 = load i32, ptr %arrayidx275, align 4
+  %or951 = or i32 %5, %1
+  %cmp282 = icmp uge i32 %mul238, %mul100
+  %cmp285 = icmp uge i32 %mul238, %mul101
+  %add28711 = or i1 %cmp282, %cmp285
+  %cmp288 = icmp uge i32 %mul238, %mul102
+  %add29012 = or i1 %add28711, %cmp288
+  %cmp291 = icmp uge i32 %mul238, %mul103
+  %add29313 = or i1 %add29012, %cmp291
+  %add293 = zext i1 %add29313 to i64
+  %arrayidx298 = getelementptr [8 x i32], ptr %p2, i64 0, i64 %add293
+  %6 = load i32, ptr %arrayidx298, align 4
+  %or301952 = or i32 %6, %2
+  %cmp310 = icmp uge i32 %mul245, %mul101
+  %cmp316 = icmp uge i32 %mul245, %mul103
+  %add31814 = or i1 %cmp310, %cmp316
+  %add318 = zext i1 %add31814 to i64
+  %arrayidx323 = getelementptr [8 x i32], ptr %p4, i64 0, i64 %add318
+  %7 = load i32, ptr %arrayidx323, align 4
+  %or326953 = or i32 %7, %3
+  %cmp332 = icmp uge i32 %mul252, %mul100
+  %cmp335 = icmp uge i32 %mul252, %mul101
+  %add33715 = or i1 %cmp332, %cmp335
+  %cmp338 = icmp uge i32 %mul252, %mul102
+  %add34016 = or i1 %add33715, %cmp338
+  %cmp341 = icmp uge i32 %mul252, %mul103
+  %add34317 = or i1 %add34016, %cmp341
+  %add343 = zext i1 %add34317 to i64
+  %arrayidx348 = getelementptr [8 x i32], ptr %p3, i64 0, i64 %add343
+  %8 = load i32, ptr %arrayidx348, align 4
+  %or351954 = or i32 %8, %4
+  %conv484 = zext i8 %0 to i32
+  %mul485 = mul i32 %conv484, 14
+  %conv491 = zext i8 %0 to i32
+  %mul492 = mul i32 %conv491, 14
+  %conv498 = zext i8 %0 to i32
+  %mul499 = mul i32 %conv498, 14
+  %conv505 = zext i8 %0 to i32
+  %mul506 = mul i32 %conv505, 14
+  %cmp519 = icmp uge i32 %mul485, %mul102
+  %cmp522 = icmp uge i32 %mul485, %mul103
+  %add52418 = or i1 %cmp519, %cmp522
+  %add524 = zext i1 %add52418 to i64
+  %arrayidx529 = getelementptr [8 x i32], ptr %p1, i64 0, i64 %add524
+  %9 = load i32, ptr %arrayidx529, align 4
+  %cmp541 = icmp uge i32 %mul492, %mul101
+  %cmp544 = icmp uge i32 %mul492, %mul102
+  %add54619 = or i1 %cmp541, %cmp544
+  %cmp547 = icmp uge i32 %mul492, %mul103
+  %add54920 = or i1 %add54619, %cmp547
+  %add549 = zext i1 %add54920 to i64
+  %arrayidx554 = getelementptr [8 x i32], ptr %p2, i64 0, i64 %add549
+  %10 = load i32, ptr %arrayidx554, align 4
+  %cmp572 = icmp uge i32 %mul499, %mul103
+  %conv573 = zext i1 %cmp572 to i64
+  %arrayidx579 = getelementptr [8 x i32], ptr %p4, i64 0, i64 %conv573
+  %11 = load i32, ptr %arrayidx579, align 4
+  %cmp594 = icmp uge i32 %mul506, %mul102
+  %conv595 = zext i1 %cmp594 to i64
+  %arrayidx604 = getelementptr [8 x i32], ptr %p3, i64 0, i64 %conv595
+  %12 = load i32, ptr %arrayidx604, align 4
+  %or4791159 = or i32 %or301952, %or951
+  %or6071160 = or i32 %or4791159, %or326953
+  %or4541161 = or i32 %or6071160, %or351954
+  %shl58111621163 = or i32 %11, %12
+  %shl55611641165 = or i32 %10, %shl58111621163
+  %shl53111661167 = or i32 %9, %shl55611641165
+  %shl5311166 = trunc i32 %shl53111661167 to i8
+  %conv616 = trunc i32 %or4541161 to i8
+  %arrayidx617 = getelementptr i8, ptr %pDst, i64 4
+  store i8 %conv616, ptr %arrayidx617, align 1
+  store i8 %shl5311166, ptr %pDst, align 1
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minbw-multiused-from-gather.ll b/llvm/test/Transforms/SLPVectorizer/X86/minbw-multiused-from-gather.ll
new file mode 100644
index 0000000000000..8e4b280271051
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minbw-multiused-from-gather.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i1 @test() {
+; CHECK-LABEL: define i1 @test() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[CONV85_22_I333_I_I:%.*]] = or i32 0, [[TMP0]]
+; CHECK-NEXT:    [[CMP3_I_22_I334_I_I:%.*]] = icmp ugt i32 [[CONV85_22_I333_I_I]], 0
+; CHECK-NEXT:    [[SHL_I111_22_I335_I_I:%.*]] = select i1 [[CMP3_I_22_I334_I_I]], i32 0, i32 0
+; CHECK-NEXT:    [[C22_I336_I_I:%.*]] = shl i32 [[CONV85_22_I333_I_I]], [[SHL_I111_22_I335_I_I]]
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[CONV85_23_I340_I_I:%.*]] = or i32 0, [[TMP1]]
+; CHECK-NEXT:    [[CMP3_I_23_I341_I_I:%.*]] = icmp ugt i32 [[CONV85_23_I340_I_I]], 0
+; CHECK-NEXT:    [[SHL_I111_23_I342_I_I:%.*]] = select i1 [[CMP3_I_23_I341_I_I]], i32 0, i32 0
+; CHECK-NEXT:    [[C23_I343_I_I:%.*]] = shl i32 [[CONV85_23_I340_I_I]], [[SHL_I111_23_I342_I_I]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[CONV85_24_I347_I_I:%.*]] = or i32 0, [[TMP2]]
+; CHECK-NEXT:    [[CMP3_I_24_I348_I_I:%.*]] = icmp ugt i32 [[CONV85_24_I347_I_I]], 0
+; CHECK-NEXT:    [[SHL_I111_24_I349_I_I:%.*]] = select i1 [[CMP3_I_24_I348_I_I]], i32 0, i32 0
+; CHECK-NEXT:    [[C24_I350_I_I:%.*]] = shl i32 [[CONV85_24_I347_I_I]], [[SHL_I111_24_I349_I_I]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 0 to i32
+; CHECK-NEXT:    [[CONV85_25_I354_I_I:%.*]] = or i32 0, [[TMP3]]
+; CHECK-NEXT:    [[CMP3_I_25_I355_I_I:%.*]] = icmp ugt i32 [[CONV85_25_I354_I_I]], 0
+; CHECK-NEXT:    [[SHL_I111_25_I356_I_I:%.*]] = select i1 [[CMP3_I_25_I355_I_I]], i32 0, i32 0
+; CHECK-NEXT:    [[C25_I357_I_I:%.*]] = shl i32 [[CONV85_25_I354_I_I]], [[SHL_I111_25_I356_I_I]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[OP_RDX:%.*]] = and i32 [[TMP4]], [[C22_I336_I_I]]
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = and i32 [[C23_I343_I_I]], [[C24_I350_I_I]]
+; CHECK-NEXT:    [[OP_RDX2:%.*]] = and i32 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT:    [[OP_RDX3:%.*]] = and i32 [[OP_RDX2]], [[C25_I357_I_I]]
+; CHECK-NEXT:    [[CONV109_I_I:%.*]] = trunc i32 [[OP_RDX3]] to i8
+; CHECK-NEXT:    [[CMP_I_I54_I:%.*]] = icmp eq i8 [[CONV109_I_I]], 0
+; CHECK-NEXT:    ret i1 [[CMP_I_I54_I]]
+;
+entry:
+  %c18.i308.i.i = shl i32 0, 0
+  %c19.i315.i.i = shl i32 0, 0
+  %and.19.i316.i.i = and i32 %c18.i308.i.i, %c19.i315.i.i
+  %c20.i322.i.i = shl i32 0, 0
+  %and.20.i323.i.i = and i32 %and.19.i316.i.i, %c20.i322.i.i
+  %c21.i329.i.i = shl i32 0, 0
+  %and.21.i330.i.i = and i32 %and.20.i323.i.i, %c21.i329.i.i
+  %0 = trunc i64 0 to i32
+  %conv85.22.i333.i.i = or i32 0, %0
+  %cmp3.i.22.i334.i.i = icmp ugt i32 %conv85.22.i333.i.i, 0
+  %shl.i111.22.i335.i.i = select i1 %cmp3.i.22.i334.i.i, i32 0, i32 0
+  %c22.i336.i.i = shl i32 %conv85.22.i333.i.i, %shl.i111.22.i335.i.i
+  %and.22.i337.i.i = and i32 %and.21.i330.i.i, %c22.i336.i.i
+  %1 = trunc i64 0 to i32
+  %conv85.23.i340.i.i = or i32 0, %1
+  %cmp3.i.23.i341.i.i = icmp ugt i32 %conv85.23.i340.i.i, 0
+  %shl.i111.23.i342.i.i = select i1 %cmp3.i.23.i341.i.i, i32 0, i32 0
+  %c23.i343.i.i = shl i32 %conv85.23.i340.i.i, %shl.i111.23.i342.i.i
+  %and.23.i344.i.i = and i32 %and.22.i337.i.i, %c23.i343.i.i
+  %2 = trunc i64 0 to i32
+  %conv85.24.i347.i.i = or i32 0, %2
+  %cmp3.i.24.i348.i.i = icmp ugt i32 %conv85.24.i347.i.i, 0
+  %shl.i111.24.i349.i.i = select i1 %cmp3.i.24.i348.i.i, i32 0, i32 0
+  %c24.i350.i.i = shl i32 %conv85.24.i347.i.i, %shl.i111.24.i349.i.i
+  %and.24.i351.i.i = and i32 %and.23.i344.i.i, %c24.i350.i.i
+  %3 = trunc i64 0 to i32
+  %conv85.25.i354.i.i = or i32 0, %3
+  %cmp3.i.25.i355.i.i = icmp ugt i32 %conv85.25.i354.i.i, 0
+  %shl.i111.25.i356.i.i = select i1 %cmp3.i.25.i355.i.i, i32 0, i32 0
+  %c25.i357.i.i = shl i32 %conv85.25.i354.i.i, %shl.i111.25.i356.i.i
+  %and.25.i358.i.i = and i32 %and.24.i351.i.i, %c25.i357.i.i
+  %conv109.i.i = trunc i32 %and.25.i358.i.i to i8
+  %cmp.i.i54.i = icmp eq i8 %conv109.i.i, 0
+  ret i1 %cmp.i.i54.i
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-num-elems.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-num-elems.ll
new file mode 100644
index 0000000000000..893dc5b3ea44d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-num-elems.ll
@@ -0,0 +1,183 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64--linux-gnu < %s -mcpu=sapphirerapids | FileCheck %s
+
+define void @test(ptr %src, ptr %dst, ptr %c, i64 %arrayidx14.15p) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], ptr [[C:%.*]], i64 [[ARRAYIDX14_15P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[ARRAYIDX12_5:%.*]] = getelementptr i8, ptr [[C]], i64 192
+; CHECK-NEXT:    [[ARRAYIDX14_5:%.*]] = getelementptr i8, ptr [[C]], i64 320
+; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr i8, ptr [[C]], i64 72
+; CHECK-NEXT:    [[ARRAYIDX12_6:%.*]] = getelementptr i8, ptr [[C]], i64 200
+; CHECK-NEXT:    [[ARRAYIDX14_6:%.*]] = getelementptr i8, ptr [[C]], i64 328
+; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr i8, ptr [[C]], i64 80
+; CHECK-NEXT:    [[ARRAYIDX12_7:%.*]] = getelementptr i8, ptr [[C]], i64 208
+; CHECK-NEXT:    [[ARRAYIDX14_7:%.*]] = getelementptr i8, ptr [[C]], i64 336
+; CHECK-NEXT:    [[ARRAYIDX12_8:%.*]] = getelementptr i8, ptr [[C]], i64 216
+; CHECK-NEXT:    [[ARRAYIDX14_8:%.*]] = getelementptr i8, ptr [[C]], i64 344
+; CHECK-NEXT:    [[ARRAYIDX_9:%.*]] = getelementptr i8, ptr [[C]], i64 96
+; CHECK-NEXT:    [[ARRAYIDX12_9:%.*]] = getelementptr i8, ptr [[C]], i64 224
+; CHECK-NEXT:    [[ARRAYIDX14_9:%.*]] = getelementptr i8, ptr [[C]], i64 352
+; CHECK-NEXT:    [[ARRAYIDX_10:%.*]] = getelementptr i8, ptr [[C]], i64 104
+; CHECK-NEXT:    [[ARRAYIDX12_10:%.*]] = getelementptr i8, ptr [[C]], i64 232
+; CHECK-NEXT:    [[ARRAYIDX14_10:%.*]] = getelementptr i8, ptr [[C]], i64 360
+; CHECK-NEXT:    [[ARRAYIDX_11:%.*]] = getelementptr i8, ptr [[C]], i64 112
+; CHECK-NEXT:    [[ARRAYIDX12_11:%.*]] = getelementptr i8, ptr [[C]], i64 240
+; CHECK-NEXT:    [[ARRAYIDX14_11:%.*]] = getelementptr i8, ptr [[C]], i64 368
+; CHECK-NEXT:    [[ARRAYIDX_12:%.*]] = getelementptr i8, ptr [[C]], i64 120
+; CHECK-NEXT:    [[ARRAYIDX12_12:%.*]] = getelementptr i8, ptr [[C]], i64 248
+; CHECK-NEXT:    [[ARRAYIDX14_12:%.*]] = getelementptr i8, ptr [[C]], i64 376
+; CHECK-NEXT:    [[ARRAYIDX14_13:%.*]] = getelementptr i8, ptr [[C]], i64 384
+; CHECK-NEXT:    [[ARRAYIDX12_5P:%.*]] = load i64, ptr [[ARRAYIDX12_5]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_5P:%.*]] = load i64, ptr [[ARRAYIDX14_5]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_6P:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_6P:%.*]] = load i64, ptr [[ARRAYIDX12_6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_6P:%.*]] = load i64, ptr [[ARRAYIDX14_6]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_7P:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_7P:%.*]] = load i64, ptr [[ARRAYIDX12_7]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_7P:%.*]] = load i64, ptr [[ARRAYIDX14_7]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_8P:%.*]] = load i64, ptr [[C]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_8P:%.*]] = load i64, ptr [[ARRAYIDX12_8]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_8P:%.*]] = load i64, ptr [[ARRAYIDX14_8]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_9P:%.*]] = load i64, ptr [[ARRAYIDX_9]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_9P:%.*]] = load i64, ptr [[ARRAYIDX12_9]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_9P:%.*]] = load i64, ptr [[ARRAYIDX14_9]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_10P:%.*]] = load i64, ptr [[ARRAYIDX_10]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_10P:%.*]] = load i64, ptr [[ARRAYIDX12_10]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_10P:%.*]] = load i64, ptr [[ARRAYIDX14_10]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_11P:%.*]] = load i64, ptr [[ARRAYIDX_11]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_11P:%.*]] = load i64, ptr [[ARRAYIDX12_11]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_11P:%.*]] = load i64, ptr [[ARRAYIDX14_11]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_12P:%.*]] = load i64, ptr [[ARRAYIDX_12]], align 8
+; CHECK-NEXT:    [[ARRAYIDX12_12P:%.*]] = load i64, ptr [[ARRAYIDX12_12]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_12P:%.*]] = load i64, ptr [[ARRAYIDX14_12]], align 8
+; CHECK-NEXT:    [[ARRAYIDX_13P:%.*]] = load i64, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[ARRAYIDX14_13P:%.*]] = load i64, ptr [[ARRAYIDX14_13]], align 8
+; CHECK-NEXT:    br label %[[LAND_LHS_TRUE137:.*]]
+; CHECK:       [[LAND_LHS_TRUE137]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_15P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, %[[LAND_LHS_TRUE137]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi i64 [ 1, %[[LAND_LHS_TRUE137]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ [[TMP0]], %[[LAND_LHS_TRUE137]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i64 [ [[TMP1]], %[[LAND_LHS_TRUE137]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi i64 [ [[TMP2]], %[[LAND_LHS_TRUE137]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi i64 [ [[TMP3]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_13P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ [[TMP4]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_13P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi i64 [ [[TMP5]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_13P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ [[TMP6]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_12P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = phi i64 [ [[TMP7]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_12P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i64 [ [[TMP8]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_12P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP9]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_11P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i64 [ [[TMP10]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_11P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi i64 [ [[TMP11]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_11P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP15:%.*]] = phi i64 [ [[TMP12]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_10P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = phi i64 [ [[TMP13]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_10P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = phi i64 [ [[TMP14]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_10P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = phi i64 [ [[TMP15]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_9P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi i64 [ [[TMP16]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_9P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi i64 [ [[TMP17]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_9P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = phi i64 [ [[TMP18]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_8P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP22:%.*]] = phi i64 [ [[TMP19]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_8P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi i64 [ [[TMP20]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_8P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = phi i64 [ [[TMP21]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_7P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = phi i64 [ [[TMP22]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_7P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = phi i64 [ [[TMP23]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_7P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = phi i64 [ [[TMP24]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_6P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP28:%.*]] = phi i64 [ [[TMP25]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_6P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = phi i64 [ [[TMP26]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX_6P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = phi i64 [ [[TMP27]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX14_5P]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[TMP31:%.*]] = phi i64 [ [[TMP28]], %[[LAND_LHS_TRUE137]] ], [ [[ARRAYIDX12_5P]], %[[ENTRY]] ]
+; CHECK-NEXT:    store i64 [[TMP31]], ptr [[DST]], align 8
+; CHECK-NEXT:    store i64 [[TMP30]], ptr [[SRC]], align 8
+; CHECK-NEXT:    store i64 [[TMP29]], ptr [[DST]], align 8
+; CHECK-NEXT:    br label %[[LAND_LHS_TRUE137]]
+;
+entry:
+  %arrayidx12.5 = getelementptr i8, ptr %c, i64 192
+  %arrayidx14.5 = getelementptr i8, ptr %c, i64 320
+  %arrayidx.6 = getelementptr i8, ptr %c, i64 72
+  %arrayidx12.6 = getelementptr i8, ptr %c, i64 200
+  %arrayidx14.6 = getelementptr i8, ptr %c, i64 328
+  %arrayidx.7 = getelementptr i8, ptr %c, i64 80
+  %arrayidx12.7 = getelementptr i8, ptr %c, i64 208
+  %arrayidx14.7 = getelementptr i8, ptr %c, i64 336
+  %arrayidx12.8 = getelementptr i8, ptr %c, i64 216
+  %arrayidx14.8 = getelementptr i8, ptr %c, i64 344
+  %arrayidx.9 = getelementptr i8, ptr %c, i64 96
+  %arrayidx12.9 = getelementptr i8, ptr %c, i64 224
+  %arrayidx14.9 = getelementptr i8, ptr %c, i64 352
+  %arrayidx.10 = getelementptr i8, ptr %c, i64 104
+  %arrayidx12.10 = getelementptr i8, ptr %c, i64 232
+  %arrayidx14.10 = getelementptr i8, ptr %c, i64 360
+  %arrayidx.11 = getelementptr i8, ptr %c, i64 112
+  %arrayidx12.11 = getelementptr i8, ptr %c, i64 240
+  %arrayidx14.11 = getelementptr i8, ptr %c, i64 368
+  %arrayidx.12 = getelementptr i8, ptr %c, i64 120
+  %arrayidx12.12 = getelementptr i8, ptr %c, i64 248
+  %arrayidx14.12 = getelementptr i8, ptr %c, i64 376
+  %arrayidx14.13 = getelementptr i8, ptr %c, i64 384
+  %arrayidx12.5p = load i64, ptr %arrayidx12.5, align 8
+  %arrayidx14.5p = load i64, ptr %arrayidx14.5, align 8
+  %arrayidx.6p = load i64, ptr %arrayidx.6, align 8
+  %arrayidx12.6p = load i64, ptr %arrayidx12.6, align 8
+  %arrayidx14.6p = load i64, ptr %arrayidx14.6, align 8
+  %arrayidx.7p = load i64, ptr %arrayidx.7, align 8
+  %arrayidx12.7p = load i64, ptr %arrayidx12.7, align 8
+  %arrayidx14.7p = load i64, ptr %arrayidx14.7, align 8
+  %arrayidx.8p = load i64, ptr %c, align 8
+  %arrayidx12.8p = load i64, ptr %arrayidx12.8, align 8
+  %arrayidx14.8p = load i64, ptr %arrayidx14.8, align 8
+  %arrayidx.9p = load i64, ptr %arrayidx.9, align 8
+  %arrayidx12.9p = load i64, ptr %arrayidx12.9, align 8
+  %arrayidx14.9p = load i64, ptr %arrayidx14.9, align 8
+  %arrayidx.10p = load i64, ptr %arrayidx.10, align 8
+  %arrayidx12.10p = load i64, ptr %arrayidx12.10, align 8
+  %arrayidx14.10p = load i64, ptr %arrayidx14.10, align 8
+  %arrayidx.11p = load i64, ptr %arrayidx.11, align 8
+  %arrayidx12.11p = load i64, ptr %arrayidx12.11, align 8
+  %arrayidx14.11p = load i64, ptr %arrayidx14.11, align 8
+  %arrayidx.12p = load i64, ptr %arrayidx.12, align 8
+  %arrayidx12.12p = load i64, ptr %arrayidx12.12, align 8
+  %arrayidx14.12p = load i64, ptr %arrayidx14.12, align 8
+  %arrayidx.13p = load i64, ptr %src, align 8
+  %arrayidx14.13p = load i64, ptr %arrayidx14.13, align 8
+  br label %land.lhs.true137
+
+land.lhs.true137:
+  %0 = phi i64 [ 0, %land.lhs.true137 ], [ %arrayidx14.15p, %entry ]
+  %1 = phi i64 [ 0, %land.lhs.true137 ], [ 1, %entry ]
+  %2 = phi i64 [ 1, %land.lhs.true137 ], [ 0, %entry ]
+  %3 = phi i64 [ %0, %land.lhs.true137 ], [ 0, %entry ]
+  %4 = phi i64 [ %1, %land.lhs.true137 ], [ 0, %entry ]
+  %5 = phi i64 [ %2, %land.lhs.true137 ], [ 0, %entry ]
+  %6 = phi i64 [ %3, %land.lhs.true137 ], [ %arrayidx14.13p, %entry ]
+  %7 = phi i64 [ %4, %land.lhs.true137 ], [ %arrayidx.13p, %entry ]
+  %8 = phi i64 [ %5, %land.lhs.true137 ], [ %arrayidx.13p, %entry ]
+  %9 = phi i64 [ %6, %land.lhs.true137 ], [ %arrayidx14.12p, %entry ]
+  %10 = phi i64 [ %7, %land.lhs.true137 ], [ %arrayidx12.12p, %entry ]
+  %11 = phi i64 [ %8, %land.lhs.true137 ], [ %arrayidx.12p, %entry ]
+  %12 = phi i64 [ %9, %land.lhs.true137 ], [ %arrayidx14.11p, %entry ]
+  %13 = phi i64 [ %10, %land.lhs.true137 ], [ %arrayidx12.11p, %entry ]
+  %14 = phi i64 [ %11, %land.lhs.true137 ], [ %arrayidx.11p, %entry ]
+  %15 = phi i64 [ %12, %land.lhs.true137 ], [ %arrayidx14.10p, %entry ]
+  %16 = phi i64 [ %13, %land.lhs.true137 ], [ %arrayidx12.10p, %entry ]
+  %17 = phi i64 [ %14, %land.lhs.true137 ], [ %arrayidx.10p, %entry ]
+  %18 = phi i64 [ %15, %land.lhs.true137 ], [ %arrayidx14.9p, %entry ]
+  %19 = phi i64 [ %16, %land.lhs.true137 ], [ %arrayidx12.9p, %entry ]
+  %20 = phi i64 [ %17, %land.lhs.true137 ], [ %arrayidx.9p, %entry ]
+  %21 = phi i64 [ %18, %land.lhs.true137 ], [ %arrayidx14.8p, %entry ]
+  %22 = phi i64 [ %19, %land.lhs.true137 ], [ %arrayidx12.8p, %entry ]
+  %23 = phi i64 [ %20, %land.lhs.true137 ], [ %arrayidx.8p, %entry ]
+  %24 = phi i64 [ %21, %land.lhs.true137 ], [ %arrayidx14.7p, %entry ]
+  %25 = phi i64 [ %22, %land.lhs.true137 ], [ %arrayidx12.7p, %entry ]
+  %26 = phi i64 [ %23, %land.lhs.true137 ], [ %arrayidx.7p, %entry ]
+  %27 = phi i64 [ %24, %land.lhs.true137 ], [ %arrayidx14.6p, %entry ]
+  %28 = phi i64 [ %25, %land.lhs.true137 ], [ %arrayidx12.6p, %entry ]
+  %29 = phi i64 [ %26, %land.lhs.true137 ], [ %arrayidx.6p, %entry ]
+  %30 = phi i64 [ %27, %land.lhs.true137 ], [ %arrayidx14.5p, %entry ]
+  %31 = phi i64 [ %28, %land.lhs.true137 ], [ %arrayidx12.5p, %entry ]
+  store i64 %31, ptr %dst, align 8
+  store i64 %30, ptr %src, align 8
+  store i64 %29, ptr %dst, align 8
+  br label %land.lhs.true137
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
new file mode 100644
index 0000000000000..47dd84c7f6e9c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-1000 < %s | FileCheck %s
+
+define void @e(ptr %c, i64 %0) {
+; CHECK-LABEL: define void @e(
+; CHECK-SAME: ptr [[C:%.*]], i64 [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[C]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i8, ptr [[TMP1]], i64 96
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 112
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[C]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x ptr>, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x ptr> [[TMP5]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <6 x ptr> poison, ptr [[TMP2]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <6 x ptr> [[TMP7]], ptr [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call <6 x ptr> @llvm.vector.insert.v6p0.v2p0(<6 x ptr> [[TMP8]], <2 x ptr> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP10:%.*]] = call <6 x ptr> @llvm.vector.insert.v6p0.v2p0(<6 x ptr> [[TMP9]], <2 x ptr> [[TMP6]], i64 4)
+; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint <6 x ptr> [[TMP10]] to <6 x i64>
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <6 x i64> [[TMP11]], <6 x i64> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <32 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <32 x i64> [[TMP13]], <32 x i64> poison, <32 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i64> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ult <32 x i64> [[TMP15]], <i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16, i64 16>
+; CHECK-NEXT:    [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP16]])
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[FOR_BODY:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %1 = load ptr, ptr %c, align 8
+  %arrayidx = getelementptr i8, ptr %1, i64 96
+  %arrayidx1 = getelementptr i8, ptr %1, i64 112
+  %2 = load ptr, ptr %arrayidx1, align 8
+  %arrayidx5 = getelementptr i8, ptr %1, i64 104
+  %3 = load ptr, ptr %arrayidx5, align 8
+  %4 = load ptr, ptr %arrayidx, align 8
+  %5 = load ptr, ptr %c, align 8
+  %6 = ptrtoint ptr %5 to i64
+  %7 = ptrtoint ptr %5 to i64
+  %8 = ptrtoint ptr %1 to i64
+  %9 = ptrtoint ptr %4 to i64
+  %10 = ptrtoint ptr %3 to i64
+  %11 = ptrtoint ptr %2 to i64
+  %12 = or i64 %0, %11
+  %dc64 = icmp ult i64 %12, 16
+  %13 = or i64 %0, %11
+  %dc65 = icmp ult i64 %13, 16
+  %cr66 = or i1 %dc64, %dc65
+  %14 = or i64 %0, %11
+  %dc67 = icmp ult i64 %14, 16
+  %cr68 = or i1 %cr66, %dc67
+  %15 = or i64 %0, %11
+  %dc69 = icmp ult i64 %15, 16
+  %cr70 = or i1 %cr68, %dc69
+  %16 = or i64 %0, %11
+  %dc71 = icmp ult i64 %16, 16
+  %cr72 = or i1 %cr70, %dc71
+  %17 = or i64 %0, %11
+  %dc73 = icmp ult i64 %17, 16
+  %cr74 = or i1 %cr72, %dc73
+  %18 = or i64 %0, %11
+  %dc75 = icmp ult i64 %18, 16
+  %cr76 = or i1 %cr74, %dc75
+  %19 = or i64 %0, %10
+  %dc77 = icmp ult i64 %19, 16
+  %cr78 = or i1 %cr76, %dc77
+  %20 = or i64 %0, %10
+  %dc79 = icmp ult i64 %20, 16
+  %cr80 = or i1 %cr78, %dc79
+  %21 = or i64 %0, %10
+  %dc81 = icmp ult i64 %21, 16
+  %cr82 = or i1 %cr80, %dc81
+  %22 = or i64 %0, %10
+  %dc83 = icmp ult i64 %22, 16
+  %cr84 = or i1 %cr82, %dc83
+  %23 = or i64 %0, %10
+  %dc85 = icmp ult i64 %23, 16
+  %cr86 = or i1 %cr84, %dc85
+  %24 = or i64 %0, %10
+  %dc87 = icmp ult i64 %24, 16
+  %cr88 = or i1 %cr86, %dc87
+  %25 = or i64 %0, %10
+  %dc89 = icmp ult i64 %25, 16
+  %cr90 = or i1 %cr88, %dc89
+  %26 = or i64 %0, %9
+  %dc91 = icmp ult i64 %26, 16
+  %cr92 = or i1 %cr90, %dc91
+  %27 = or i64 %0, %9
+  %dc93 = icmp ult i64 %27, 16
+  %cr94 = or i1 %cr92, %dc93
+  %28 = or i64 %0, %9
+  %dc95 = icmp ult i64 %28, 16
+  %cr96 = or i1 %cr94, %dc95
+  %29 = or i64 %0, %9
+  %dc97 = icmp ult i64 %29, 16
+  %cr98 = or i1 %cr96, %dc97
+  %30 = or i64 %0, %9
+  %dc99 = icmp ult i64 %30, 16
+  %cr100 = or i1 %cr98, %dc99
+  %31 = or i64 %0, %9
+  %dc101 = icmp ult i64 %31, 16
+  %cr102 = or i1 %cr100, %dc101
+  %32 = or i64 %0, %8
+  %dc103 = icmp ult i64 %32, 16
+  %cr104 = or i1 %cr102, %dc103
+  %33 = or i64 %0, %8
+  %dc105 = icmp ult i64 %33, 16
+  %cr106 = or i1 %cr104, %dc105
+  %34 = or i64 %0, %8
+  %dc107 = icmp ult i64 %34, 16
+  %cr108 = or i1 %cr106, %dc107
+  %35 = or i64 %0, %8
+  %dc109 = icmp ult i64 %35, 16
+  %cr110 = or i1 %cr108, %dc109
+  %36 = or i64 %0, %8
+  %dc111 = icmp ult i64 %36, 16
+  %cr112 = or i1 %cr110, %dc111
+  %37 = or i64 %0, %7
+  %dc113 = icmp ult i64 %37, 16
+  %cr114 = or i1 %cr112, %dc113
+  %38 = or i64 %0, %7
+  %dc115 = icmp ult i64 %38, 16
+  %cr116 = or i1 %cr114, %dc115
+  %39 = or i64 %0, %7
+  %dc117 = icmp ult i64 %39, 16
+  %cr118 = or i1 %cr116, %dc117
+  %40 = or i64 %0, %7
+  %dc119 = icmp ult i64 %40, 16
+  %cr120 = or i1 %cr118, %dc119
+  %41 = or i64 %0, %6
+  %dc121 = icmp ult i64 %41, 16
+  %cr122 = or i1 %cr120, %dc121
+  %42 = or i64 %0, %6
+  %dc123 = icmp ult i64 %42, 16
+  %cr124 = or i1 %cr122, %dc123
+  %43 = or i64 %0, %6
+  %dc125 = icmp ult i64 %43, 16
+  %cr126 = or i1 %cr124, %dc125
+  br i1 %cr126, label %for.body, label %vector.ph
+
+vector.ph:
+  ret void
+
+for.body:
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/reduction-whole-regs-loads.ll b/llvm/test/Transforms/SLPVectorizer/reduction-whole-regs-loads.ll
index 281b5f99540ea..4074b8654362e 100644
--- a/llvm/test/Transforms/SLPVectorizer/reduction-whole-regs-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/reduction-whole-regs-loads.ll
@@ -1,21 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=riscv64-unknown-linux -mattr=+v -slp-threshold=-100 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=riscv64-unknown-linux -mattr=+v -slp-threshold=-100 | FileCheck %s --check-prefix=RISCV
 ; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux -slp-threshold=-100 | FileCheck %s
 ; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux -slp-threshold=-100 | FileCheck %s
 ; REQUIRES: aarch64-registered-target, x86-registered-target, riscv-registered-target
 
 define i64 @test(ptr %p) {
+; RISCV-LABEL: @test(
+; RISCV-NEXT:  entry:
+; RISCV-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 4
+; RISCV-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[P]], align 4
+; RISCV-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_4]], align 4
+; RISCV-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0>
+; RISCV-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.vector.insert.v8i64.v4i64(<8 x i64> [[TMP2]], <4 x i64> [[TMP0]], i64 0)
+; RISCV-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.vector.insert.v8i64.v2i64(<8 x i64> [[TMP3]], <2 x i64> [[TMP1]], i64 4)
+; RISCV-NEXT:    [[TMP5:%.*]] = mul <8 x i64> [[TMP4]], <i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42>
+; RISCV-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
+; RISCV-NEXT:    ret i64 [[TMP6]]
+;
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 4
-; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i64>, ptr [[P]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[ARRAYIDX_4]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0>
-; CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.vector.insert.v8i64.v4i64(<8 x i64> [[TMP2]], <4 x i64> [[TMP0]], i64 0)
-; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i64> @llvm.vector.insert.v8i64.v2i64(<8 x i64> [[TMP3]], <2 x i64> [[TMP1]], i64 4)
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <8 x i64> [[TMP4]], <i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42>
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP5]])
-; CHECK-NEXT:    ret i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <6 x i64>, ptr [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x i64> [[TMP0]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i64> [[TMP1]], <i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42, i64 42>
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[TMP2]])
+; CHECK-NEXT:    ret i64 [[TMP3]]
 ;
 entry:
   %arrayidx.1 = getelementptr inbounds i64, ptr %p, i64 1
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-store.ll b/llvm/test/Transforms/SimplifyCFG/speculate-store.ll
index c7ebeff5246d6..d6da9fd8ae20c 100644
--- a/llvm/test/Transforms/SimplifyCFG/speculate-store.ll
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-store.ll
@@ -194,6 +194,165 @@ if.end:
   ret i32 %add
 }
 
+define i64 @load_before_store_noescape_byval(ptr byval([2 x i32]) %a, i64 %i, i32 %b)  {
+; CHECK-LABEL: @load_before_store_noescape_byval(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 -1, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[V]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[B]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[V2]]
+;
+entry:
+  store i64 -1, ptr %a, align 8
+  %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %cmp = icmp slt i32 %v, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %b, ptr %arrayidx, align 4
+  br label %if.end
+
+if.end:
+  %v2 = load i64, ptr %a, align 8
+  ret i64 %v2
+}
+
+declare noalias ptr @malloc(i64 %size)
+
+define i64 @load_before_store_noescape_malloc(i64 %i, i32 %b)  {
+; CHECK-LABEL: @load_before_store_noescape_malloc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = call ptr @malloc(i64 8)
+; CHECK-NEXT:    store i64 -1, ptr [[A]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 0, i64 [[I:%.*]]
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[V]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[B]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[V2]]
+;
+entry:
+  %a = call ptr @malloc(i64 8)
+  store i64 -1, ptr %a, align 8
+  %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
+  %v = load i32, ptr %arrayidx, align 4
+  %cmp = icmp slt i32 %v, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %b, ptr %arrayidx, align 4
+  br label %if.end
+
+if.end:
+  %v2 = load i64, ptr %a, align 8
+  ret i64 %v2
+}
+
+define i64 @load_before_store_noescape_writable(ptr noalias writable dereferenceable(8) %a, i64 %i, i32 %b)  {
+; CHECK-LABEL: @load_before_store_noescape_writable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 -1, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[V]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[B]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[V2]]
+;
+entry:
+  store i64 -1, ptr %a, align 8
+  %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 1
+  %v = load i32, ptr %arrayidx, align 4
+  %cmp = icmp slt i32 %v, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %b, ptr %arrayidx, align 4
+  br label %if.end
+
+if.end:
+  %v2 = load i64, ptr %a, align 8
+  ret i64 %v2
+}
+
+define i64 @load_before_store_noescape_writable_missing_noalias(ptr writable dereferenceable(8) %a, i64 %i, i32 %b)  {
+; CHECK-LABEL: @load_before_store_noescape_writable_missing_noalias(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 -1, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[V]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[B]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[V2]]
+;
+entry:
+  store i64 -1, ptr %a, align 8
+  %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 1
+  %v = load i32, ptr %arrayidx, align 4
+  %cmp = icmp slt i32 %v, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %b, ptr %arrayidx, align 4
+  br label %if.end
+
+if.end:
+  %v2 = load i64, ptr %a, align 8
+  ret i64 %v2
+}
+
+define i64 @load_before_store_noescape_writable_missing_derefable(ptr noalias writable %a, i64 %i, i32 %b)  {
+; CHECK-LABEL: @load_before_store_noescape_writable_missing_derefable(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i64 -1, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[A]], i64 0, i64 1
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[V]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    store i32 [[B]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A]], align 8
+; CHECK-NEXT:    ret i64 [[V2]]
+;
+entry:
+  store i64 -1, ptr %a, align 8
+  %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 1
+  %v = load i32, ptr %arrayidx, align 4
+  %cmp = icmp slt i32 %v, %b
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:
+  store i32 %b, ptr %arrayidx, align 4
+  br label %if.end
+
+if.end:
+  %v2 = load i64, ptr %a, align 8
+  ret i64 %v2
+}
+
 declare void @fork_some_threads(ptr);
 declare void @join_some_threads();
 
diff --git a/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll b/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
index 13d13bca56de6..cb99632c287b3 100644
--- a/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
+++ b/llvm/test/Verifier/AMDGPU/intrinsic-immarg.ll
@@ -36,27 +36,27 @@ define void @struct_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %idx, i32
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 define void @invalid_image_sample_1d_v4f32_f32(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc, i32 %dmask, i1 %bool, i32 %arg) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %dmask
-  ; CHECK-NEXT: %data0 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 %dmask, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
-  %data0 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 %dmask, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  ; CHECK-NEXT: %data0 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 %dmask, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+  %data0 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 %dmask, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i1 %bool
-  ; CHECK-NEXT: %data1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 %bool, i32 0, i32 0)
-  %data1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 %bool, i32 0, i32 0)
+  ; CHECK-NEXT: %data1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 %bool, i32 0, i32 0)
+  %data1 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 %bool, i32 0, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %arg
-  ; CHECK-NEXT:   %data2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 %arg, i32 0)
-  %data2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 %arg, i32 0)
+  ; CHECK-NEXT:   %data2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 %arg, i32 0)
+  %data2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 %arg, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %arg
-  ; CHECK-NEXT:   %data3 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 %arg)
-  %data3 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 %arg)
+  ; CHECK-NEXT:   %data3 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 %arg)
+  %data3 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32.v8i32.v4i32(i32 0, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 %arg)
   ret void
 }
 
@@ -336,84 +336,84 @@ define amdgpu_kernel void @update_dpp_test(ptr addrspace(1) %out, i32 %in1, i32
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32)
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32, i32, <8 x i32>, i32, i32)
 define amdgpu_ps void @load_1d(<8 x i32> inreg %rsrc, i32 %s, i32 %var) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 %var, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
-  %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 %var, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ; CHECK-NEXT: %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 %var, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 %var, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %var, i32 0)
-  %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %var, i32 0)
+  ; CHECK-NEXT: %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %var, i32 0)
+  %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %var, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 %var)
-  %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 %var)
+  ; CHECK-NEXT: %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 %var)
+  %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 %var)
   ret void
 }
 
-declare {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32, i32, <8 x i32>, i32, i32)
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32)
 define amdgpu_ps void @load_1d_tfe(<8 x i32> inreg %rsrc, ptr addrspace(1) inreg %out, i32 %s, i32 %val) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val0 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 %val, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
-  %val0 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 %val, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  ; CHECK-NEXT: %val0 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 %val, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %val0 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32.v8i32(i32 %val, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val1 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
-  %val1 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  ; CHECK-NEXT: %val1 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  %val1 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val2 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 %val)
-  %val2 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 %val)
+  ; CHECK-NEXT: %val2 = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 %val)
+  %val2 = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32.v8i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 %val)
   ret void
 }
 
-declare {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
+declare {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32.v8i32.v4i32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
 define amdgpu_ps void @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, ptr addrspace(1) inreg %out, float %s, i32 %var, i1 %bool) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val0 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
-  %val0 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
+  ; CHECK-NEXT: %val0 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
+  %val0 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32.v8i32.v4i32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i1 %bool
-  ; CHECK-NEXT: %val1 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 %bool, i32 1, i32 0)
-  %val1 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 %bool, i32 1, i32 0)
+  ; CHECK-NEXT: %val1 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 %bool, i32 1, i32 0)
+  %val1 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32.v8i32.v4i32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 %bool, i32 1, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val2 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 %var, i32 0)
-  %val2 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 %var, i32 0)
+  ; CHECK-NEXT: %val2 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 %var, i32 0)
+  %val2 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32.v8i32.v4i32(i32 16, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 %var, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val3 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 %var)
-  %val3 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 %var)
+  ; CHECK-NEXT: %val3 = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32.v8i32.v4i32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 %var)
+  %val3 = call {<4 x float>, i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32.v8i32.v4i32(i32 %var, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 %var)
   ret void
 }
 
-declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32, i16, <8 x i32>, i32, i32)
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32, i16, <8 x i32>, i32, i32)
 define amdgpu_ps void @load_1d_a16(<8 x i32> inreg %rsrc, <2 x i16> %coords, i16 %s, i32 %var) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 %var, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
-  %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 %var, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  ; CHECK-NEXT: %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 %var, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  %val0 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 %var, i16 %s, <8 x i32> %rsrc, i32 0, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 %s, <8 x i32> %rsrc, i32 %var, i32 0)
-  %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 %s, <8 x i32> %rsrc, i32 %var, i32 0)
+  ; CHECK-NEXT: %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 %s, <8 x i32> %rsrc, i32 %var, i32 0)
+  %val1 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 %s, <8 x i32> %rsrc, i32 %var, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %var
-  ; CHECK-NEXT: %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 %var)
-  %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 %var)
+  ; CHECK-NEXT: %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 %var)
+  %val2 = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i16.v8i32(i32 15, i16 %s, <8 x i32> %rsrc, i32 0, i32 %var)
   ret void
 }
 
@@ -426,31 +426,31 @@ define amdgpu_ps void @raw_buffer_atomic_swap(<4 x i32> inreg %rsrc, i32 %data,
   ret void
 }
 
-declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32, i32, <8 x i32>, i32, i32)
+declare i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32, i32, <8 x i32>, i32, i32)
 define amdgpu_ps void @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %val) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val0 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
-  %val0 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  ; CHECK-NEXT: %val0 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  %val0 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val1 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
-  %val1 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
+  ; CHECK-NEXT: %val1 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
+  %val1 = call i32 @llvm.amdgcn.image.atomic.swap.1d.i32.i32.v8i32(i32 %data, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
   ret void
 }
 
-declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #0
+declare i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32, i32, i32, <8 x i32>, i32, i32) #0
 define amdgpu_ps void @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %swap, i32 %s, i32 %val) {
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val0 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
-  %val0 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  ; CHECK-NEXT: %val0 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
+  %val0 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 %val, i32 0)
 
   ; CHECK: immarg operand has non-immediate parameter
   ; CHECK-NEXT: i32 %val
-  ; CHECK-NEXT: %val1 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
-  %val1 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
+  ; CHECK-NEXT: %val1 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
+  %val1 = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32.v8i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 %val)
   ret void
 }
 
diff --git a/llvm/test/tools/dsymutil/X86/remarks-linking-archive.text b/llvm/test/tools/dsymutil/X86/remarks-linking-archive.text
index e23d0b620cac1..47f9df82296fe 100644
--- a/llvm/test/tools/dsymutil/X86/remarks-linking-archive.text
+++ b/llvm/test/tools/dsymutil/X86/remarks-linking-archive.text
@@ -14,9 +14,9 @@ RUN: llvm-bcanalyzer -dump %t/basic.macho.remarks.archive.x86_64.dSYM/Contents/R
 
 Check that we don't error if we're missing remark files from an archive, but we warn instead.
 Instead of creating a new binary, just remove the remarks prepend path.
-RUN: dsymutil -oso-prepend-path=%p/../Inputs %t/basic.macho.remarks.archive.x86_64 2>&1 | FileCheck %s --check-prefix=CHECK-MISSING
+RUN: dsymutil -oso-prepend-path=%p/../Inputs %t/basic.macho.remarks.archive.x86_64 2>&1 | FileCheck -DMSG=%errc_ENOENT %s --check-prefix=CHECK-MISSING
 
-RUN: dsymutil --linker parallel -oso-prepend-path=%p/../Inputs %t/basic.macho.remarks.archive.x86_64 2>&1 | FileCheck %s --check-prefix=CHECK-MISSING
+RUN: dsymutil --linker parallel -oso-prepend-path=%p/../Inputs %t/basic.macho.remarks.archive.x86_64 2>&1 | FileCheck -DMSG=%errc_ENOENT %s --check-prefix=CHECK-MISSING
 
 CHECK: <Meta
 CHECK: <Remark Num
@@ -42,9 +42,9 @@ CHECK: <Remark Num
 CHECK: <Remark Num
 CHECK-NOT: <Remark Num
 
-CHECK-MISSING: warning: '/remarks/basic1.macho.remarks.x86_64.opt.bitstream': No such file or directory
+CHECK-MISSING: warning: '/remarks/basic1.macho.remarks.x86_64.opt.bitstream': [[MSG]]
 CHECK-MISSING-NEXT: note: while processing {{.*}}libbasic.a(basic1.macho.remarks.x86_64.o)
-CHECK-MISSING-NEXT: warning: '/remarks/basic2.macho.remarks.x86_64.opt.bitstream': No such file or directory
+CHECK-MISSING-NEXT: warning: '/remarks/basic2.macho.remarks.x86_64.opt.bitstream': [[MSG]]
 CHECK-MISSING-NEXT: note: while processing {{.*}}libbasic.a(basic2.macho.remarks.x86_64.o)
-CHECK-MISSING-NEXT: warning: '/remarks/basic3.macho.remarks.x86_64.opt.bitstream': No such file or directory
+CHECK-MISSING-NEXT: warning: '/remarks/basic3.macho.remarks.x86_64.opt.bitstream': [[MSG]]
 CHECK-MISSING-NEXT: note: while processing {{.*}}libbasic.a(basic3.macho.remarks.x86_64.o)
diff --git a/llvm/test/tools/gold/X86/stats-file-option.ll b/llvm/test/tools/gold/X86/stats-file-option.ll
index a0009a7a874d7..c0c6ffd1f1080 100644
--- a/llvm/test/tools/gold/X86/stats-file-option.ll
+++ b/llvm/test/tools/gold/X86/stats-file-option.ll
@@ -21,5 +21,5 @@ define i32 @foo() {
 
 ; Try to save statistics to an invalid file.
 ; RUN: not %gold -plugin %llvmshlibdir/LLVMgold%shlibext -plugin-opt=stats-file=%t2/foo.stats \
-; RUN:    -m elf_x86_64 -r -o %t.o %t.bc 2>&1 | FileCheck --check-prefix=ERROR %s
-; ERROR: LLVM gold plugin: No such file or directory
+; RUN:    -m elf_x86_64 -r -o %t.o %t.bc 2>&1 | FileCheck -DMSG=%errc_ENOENT --check-prefix=ERROR %s
+; ERROR: LLVM gold plugin: [[MSG]]
diff --git a/llvm/test/tools/llvm-ar/read-only-archive.test b/llvm/test/tools/llvm-ar/read-only-archive.test
index 0365f6ad054da..e3659903d8cb3 100644
--- a/llvm/test/tools/llvm-ar/read-only-archive.test
+++ b/llvm/test/tools/llvm-ar/read-only-archive.test
@@ -11,12 +11,12 @@
 # RUN: chmod 444 %t/archive.a
 
 # RUN: not llvm-ar r %t/archive.a %t/3.txt \
-# RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a
+# RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a -DMSG=%errc_ENOENT
 
-# ERROR: error: [[FILE]]: no such file or directory
+# ERROR: error: [[FILE]]: [[MSG]]
 
 # RUN: not llvm-ar q %t/archive.a %t/3.txt \
-# RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a
+# RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a -DMSG=%errc_ENOENT
 
 # RUN: not llvm-ar m %t/archive.a t/1.txt \
 # RUN:   FileCheck %s --check-prefix=ERROR-2 -DFILE=%t/archive.a
diff --git a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
index 08c83c9f907fb..e8668a7f198a9 100644
--- a/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
+++ b/llvm/test/tools/llvm-ctxprof-util/llvm-ctxprof-util-negative.test
@@ -3,7 +3,7 @@
 ; RUN: not llvm-ctxprof-util nofile.json 2>&1 | FileCheck %s --check-prefix=NO_CMD
 ; RUN: not llvm-ctxprof-util invalidCmd --input nofile.json 2>&1 | FileCheck %s --check-prefix=INVALID_CMD
 ; RUN: not llvm-ctxprof-util fromJSON nofile.json 2>&1 | FileCheck %s --check-prefix=NO_FLAG
-; RUN: not llvm-ctxprof-util fromJSON --input nofile.json 2>&1 | FileCheck %s --check-prefix=NO_FILE
+; RUN: not llvm-ctxprof-util fromJSON --input nofile.json 2>&1 | FileCheck -DMSG=%errc_ENOENT %s --check-prefix=NO_FILE
 ; RUN: not llvm-ctxprof-util fromJSON --input %S/Inputs/bad.json 2>&1 | FileCheck %s --check-prefix=BAD_JSON
 ; RUN: not llvm-ctxprof-util fromJSON --input %S/Inputs/invalid-no-vector.json 2>&1 | FileCheck %s --check-prefix=NO_VECTOR
 ; RUN: not llvm-ctxprof-util fromJSON --input %S/Inputs/invalid-no-ctx.json 2>&1 | FileCheck %s --check-prefix=NO_CTX
@@ -15,7 +15,7 @@
 ; NO_CMD: Unknown subcommand 'nofile.json'
 ; INVALID_CMD: Unknown subcommand 'invalidCmd'
 ; NO_FLAG: Unknown command line argument 'nofile.json'. 
-; NO_FILE: 'nofile.json': No such file or directory
+; NO_FILE: 'nofile.json': [[MSG]]
 ; BAD_JSON: Expected object key
 ; NO_VECTOR: expected array
 ; NO_CTX: missing value at (root)[0].Guid
diff --git a/llvm/tools/llvm-dis/llvm-dis.cpp b/llvm/tools/llvm-dis/llvm-dis.cpp
index a3a62f042ddbd..744201289b4a4 100644
--- a/llvm/tools/llvm-dis/llvm-dis.cpp
+++ b/llvm/tools/llvm-dis/llvm-dis.cpp
@@ -203,7 +203,7 @@ int main(int argc, char **argv) {
     return 1;
   }
 
-  for (std::string InputFilename : InputFilenames) {
+  for (const auto &InputFilename : InputFilenames) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
         MemoryBuffer::getFileOrSTDIN(InputFilename);
     if (std::error_code EC = BufferOrErr.getError()) {
diff --git a/llvm/unittests/ADT/SmallSetTest.cpp b/llvm/unittests/ADT/SmallSetTest.cpp
index 0fb20b19df925..2feb0b1feb421 100644
--- a/llvm/unittests/ADT/SmallSetTest.cpp
+++ b/llvm/unittests/ADT/SmallSetTest.cpp
@@ -12,11 +12,63 @@
 
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <string>
 
 using namespace llvm;
 
+TEST(SmallSetTest, ConstructorIteratorPair) {
+  std::initializer_list<int> L = {1, 2, 3, 4, 5};
+  SmallSet<int, 4> S(std::begin(L), std::end(L));
+  EXPECT_THAT(S, testing::UnorderedElementsAreArray(L));
+}
+
+TEST(SmallSet, ConstructorRange) {
+  std::initializer_list<int> L = {1, 2, 3, 4, 5};
+
+  SmallSet<int, 4> S(llvm::make_range(std::begin(L), std::end(L)));
+  EXPECT_THAT(S, testing::UnorderedElementsAreArray(L));
+}
+
+TEST(SmallSet, ConstructorInitializerList) {
+  std::initializer_list<int> L = {1, 2, 3, 4, 5};
+  SmallSet<int, 4> S = {1, 2, 3, 4, 5};
+  EXPECT_THAT(S, testing::UnorderedElementsAreArray(L));
+}
+
+TEST(SmallSet, CopyConstructor) {
+  SmallSet<int, 4> S = {1, 2, 3};
+  SmallSet<int, 4> T = S;
+
+  EXPECT_THAT(S, testing::ContainerEq(T));
+}
+
+TEST(SmallSet, MoveConstructor) {
+  std::initializer_list<int> L = {1, 2, 3};
+  SmallSet<int, 4> S = L;
+  SmallSet<int, 4> T = std::move(S);
+
+  EXPECT_THAT(T, testing::UnorderedElementsAreArray(L));
+}
+
+TEST(SmallSet, CopyAssignment) {
+  SmallSet<int, 4> S = {1, 2, 3};
+  SmallSet<int, 4> T;
+  T = S;
+
+  EXPECT_THAT(S, testing::ContainerEq(T));
+}
+
+TEST(SmallSet, MoveAssignment) {
+  std::initializer_list<int> L = {1, 2, 3};
+  SmallSet<int, 4> S = L;
+  SmallSet<int, 4> T;
+  T = std::move(S);
+
+  EXPECT_THAT(T, testing::UnorderedElementsAreArray(L));
+}
+
 TEST(SmallSetTest, Insert) {
 
   SmallSet<int, 4> s1;
diff --git a/llvm/unittests/Analysis/SparsePropagation.cpp b/llvm/unittests/Analysis/SparsePropagation.cpp
index 8583100dc9bc8..ca73a480cbb2d 100644
--- a/llvm/unittests/Analysis/SparsePropagation.cpp
+++ b/llvm/unittests/Analysis/SparsePropagation.cpp
@@ -139,7 +139,8 @@ class TestLatticeFunc
   /// Compute the lattice values that change as a result of executing the given
   /// instruction. We only handle the few instructions needed for the tests.
   void ComputeInstructionState(
-      Instruction &I, DenseMap<TestLatticeKey, TestLatticeVal> &ChangedValues,
+      Instruction &I,
+      SmallDenseMap<TestLatticeKey, TestLatticeVal, 16> &ChangedValues,
       SparseSolver<TestLatticeKey, TestLatticeVal> &SS) override {
     switch (I.getOpcode()) {
     case Instruction::Call:
@@ -159,7 +160,7 @@ class TestLatticeFunc
   /// actual argument state. The call site state is the merge of the call site
   /// state with the returned value state of the called function.
   void visitCallBase(CallBase &I,
-                     DenseMap<TestLatticeKey, TestLatticeVal> &ChangedValues,
+                     SmallDenseMap<TestLatticeKey, TestLatticeVal, 16> &ChangedValues,
                      SparseSolver<TestLatticeKey, TestLatticeVal> &SS) {
     Function *F = I.getCalledFunction();
     auto RegI = TestLatticeKey(&I, IPOGrouping::Register);
@@ -183,7 +184,7 @@ class TestLatticeFunc
   /// Handle return instructions. The function's return state is the merge of
   /// the returned value state and the function's current return state.
   void visitReturn(ReturnInst &I,
-                   DenseMap<TestLatticeKey, TestLatticeVal> &ChangedValues,
+                   SmallDenseMap<TestLatticeKey, TestLatticeVal, 16> &ChangedValues,
                    SparseSolver<TestLatticeKey, TestLatticeVal> &SS) {
     Function *F = I.getParent()->getParent();
     if (F->getReturnType()->isVoidTy())
@@ -199,7 +200,7 @@ class TestLatticeFunc
   /// is the merge of the stored value state with the current global variable
   /// state.
   void visitStore(StoreInst &I,
-                  DenseMap<TestLatticeKey, TestLatticeVal> &ChangedValues,
+                  SmallDenseMap<TestLatticeKey, TestLatticeVal, 16> &ChangedValues,
                   SparseSolver<TestLatticeKey, TestLatticeVal> &SS) {
     auto *GV = dyn_cast<GlobalVariable>(I.getPointerOperand());
     if (!GV)
@@ -213,7 +214,7 @@ class TestLatticeFunc
   /// Handle all other instructions. All other instructions are marked
   /// overdefined.
   void visitInst(Instruction &I,
-                 DenseMap<TestLatticeKey, TestLatticeVal> &ChangedValues,
+                 SmallDenseMap<TestLatticeKey, TestLatticeVal, 16> &ChangedValues,
                  SparseSolver<TestLatticeKey, TestLatticeVal> &SS) {
     auto RegI = TestLatticeKey(&I, IPOGrouping::Register);
     ChangedValues[RegI] = getOverdefinedVal();
diff --git a/llvm/unittests/IR/IntrinsicsTest.cpp b/llvm/unittests/IR/IntrinsicsTest.cpp
index a92ffe3cdeb7e..0c4af28a2ab57 100644
--- a/llvm/unittests/IR/IntrinsicsTest.cpp
+++ b/llvm/unittests/IR/IntrinsicsTest.cpp
@@ -94,7 +94,6 @@ TEST(IntrinsicNameLookup, ClangBuiltinLookup) {
       {"__builtin_amdgcn_workgroup_id_z", "amdgcn", amdgcn_workgroup_id_z},
       {"__builtin_arm_cdp", "arm", arm_cdp},
       {"__builtin_bpf_preserve_type_info", "bpf", bpf_preserve_type_info},
-      {"__builtin_hlsl_create_handle", "dx", dx_create_handle},
       {"__builtin_HEXAGON_A2_tfr", "hexagon", hexagon_A2_tfr},
       {"__builtin_lasx_xbz_w", "loongarch", loongarch_lasx_xbz_w},
       {"__builtin_mips_bitrev", "mips", mips_bitrev},
diff --git a/llvm/unittests/SandboxIR/CMakeLists.txt b/llvm/unittests/SandboxIR/CMakeLists.txt
index 2ab284a511fca..622496ada567f 100644
--- a/llvm/unittests/SandboxIR/CMakeLists.txt
+++ b/llvm/unittests/SandboxIR/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(SandboxIRTests
   PassTest.cpp
+  RegionTest.cpp
   SandboxIRTest.cpp
   TrackerTest.cpp
   TypesTest.cpp
diff --git a/llvm/unittests/SandboxIR/PassTest.cpp b/llvm/unittests/SandboxIR/PassTest.cpp
index 2eaf369caf084..9cd54352735dc 100644
--- a/llvm/unittests/SandboxIR/PassTest.cpp
+++ b/llvm/unittests/SandboxIR/PassTest.cpp
@@ -9,8 +9,9 @@
 #include "llvm/SandboxIR/Pass.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Module.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
 #include "llvm/SandboxIR/PassManager.h"
-#include "llvm/SandboxIR/SandboxIR.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/RegionTest.cpp b/llvm/unittests/SandboxIR/RegionTest.cpp
similarity index 97%
rename from llvm/unittests/Transforms/Vectorize/SandboxVectorizer/RegionTest.cpp
rename to llvm/unittests/SandboxIR/RegionTest.cpp
index 0318d32c69219..f1bb535d9c50e 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/RegionTest.cpp
+++ b/llvm/unittests/SandboxIR/RegionTest.cpp
@@ -6,9 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/Region.h"
+#include "llvm/SandboxIR/Region.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 42df09609b675..66a5191b1154b 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/SandboxIR/SandboxIR.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
@@ -14,7 +13,12 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Module.h"
+#include "llvm/SandboxIR/BasicBlock.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Instruction.h"
+#include "llvm/SandboxIR/Module.h"
 #include "llvm/SandboxIR/Utils.h"
+#include "llvm/SandboxIR/Value.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
@@ -1685,6 +1689,64 @@ void @foo0(i32 %arg0, i32 %arg1) {
 #endif // NDEBUG
 }
 
+TEST_F(SandboxIRTest, Module) {
+  parseIR(C, R"IR(
+@glob0 = global i32 42
+@glob1 = global i32 43
+@internal0 = internal global i32 42
+@const0 = constant i32 42
+@alias0 = dso_local alias void(), ptr @foo
+@ifunc = ifunc void(), ptr @foo
+define void @foo() {
+  ret void
+}
+define void @bar() {
+  ret void
+}
+)IR");
+  llvm::Module *LLVMM = &*M;
+  llvm::Function *LLVMFFoo = &*M->getFunction("foo");
+  llvm::Function *LLVMFBar = &*M->getFunction("bar");
+
+  sandboxir::Context Ctx(C);
+  auto *M = Ctx.createModule(LLVMM);
+  // Check getContext().
+  EXPECT_EQ(&M->getContext(), &Ctx);
+  // Check getFunction().
+  auto *FFoo = M->getFunction("foo");
+  auto *FBar = M->getFunction("bar");
+  EXPECT_EQ(FFoo, Ctx.getValue(LLVMFFoo));
+  EXPECT_EQ(FBar, Ctx.getValue(LLVMFBar));
+  // Check getDataLayout().
+  EXPECT_EQ(&M->getDataLayout(), &LLVMM->getDataLayout());
+  // Check getSourceFileName().
+  EXPECT_EQ(M->getSourceFileName(), LLVMM->getSourceFileName());
+  // Check getGlobalVariable().
+  for (const char *Name : {"global0", "global1", "internal0"})
+    EXPECT_EQ(M->getGlobalVariable(Name),
+              Ctx.getValue(LLVMM->getGlobalVariable(Name)));
+  // Check getGlobalVariable(AllowInternal).
+  {
+    auto *Internal0 = M->getGlobalVariable("internal0", /*AllowInternal=*/true);
+    EXPECT_TRUE(Internal0 != nullptr);
+    EXPECT_EQ(Internal0, Ctx.getValue(LLVMM->getNamedGlobal("internal0")));
+  }
+  // Check getNamedGlobal().
+  {
+    auto *Internal = M->getNamedGlobal("internal0");
+    EXPECT_TRUE(Internal != nullptr);
+    EXPECT_EQ(Internal, Ctx.getValue(LLVMM->getNamedGlobal("internal0")));
+  }
+  // Check getNamedAlias().
+  auto *Alias0 = M->getNamedAlias("alias0");
+  EXPECT_EQ(Alias0, Ctx.getValue(LLVMM->getNamedAlias("alias0")));
+  EXPECT_EQ(M->getNamedAlias("aliasFOO"), nullptr);
+  // Check getNamedIFunc().
+  auto *IFunc0 = M->getNamedIFunc("ifunc0");
+  EXPECT_EQ(IFunc0, Ctx.getValue(LLVMM->getNamedAlias("ifunc0")));
+  EXPECT_EQ(M->getNamedIFunc("ifuncFOO"), nullptr);
+}
+
 TEST_F(SandboxIRTest, BasicBlock) {
   parseIR(C, R"IR(
 define void @foo(i32 %v1) {
@@ -1770,9 +1832,12 @@ define void @foo(i8 %v1, ptr %ptr) {
   %atomicrmw = atomicrmw add ptr %ptr, i8 %v1 acquire
   %udiv = udiv i8 %ld0, %v1
   %urem = urem i8 %ld0, %v1
-  call void @foo()
-  ret void
+  call void @foo(), !dbg !1
+  ret void, !tbaa !2
 }
+
+!1 = !{}
+!2 = !{}
 )IR");
   llvm::Function *LLVMF = &*M->getFunction("foo");
   llvm::BasicBlock *LLVMBB1 = getBasicBlockByName(*LLVMF, "bb1");
@@ -1806,6 +1871,15 @@ define void @foo(i8 %v1, ptr %ptr) {
   EXPECT_EQ(I1->getOpcode(), sandboxir::Instruction::Opcode::Sub);
   EXPECT_EQ(Ret->getOpcode(), sandboxir::Instruction::Opcode::Ret);
 
+  // Check getOpcodeName().
+  EXPECT_STREQ(I0->getOpcodeName(), "Add");
+  EXPECT_STREQ(I1->getOpcodeName(), "Sub");
+  EXPECT_STREQ(Ret->getOpcodeName(), "Ret");
+
+  EXPECT_STREQ(sandboxir::Instruction::getOpcodeName(
+                   sandboxir::Instruction::Opcode::Alloca),
+               "Alloca");
+
   // Check moveBefore(I).
   I1->moveBefore(I0);
   EXPECT_EQ(I0->getPrevNode(), I1);
@@ -1874,6 +1948,19 @@ define void @foo(i8 %v1, ptr %ptr) {
     EXPECT_EQ(LLVMI.isShift(), I.isShift());
     // Check isCast().
     EXPECT_EQ(LLVMI.isCast(), I.isCast());
+    // Check isFuncletPad().
+    EXPECT_EQ(LLVMI.isFuncletPad(), I.isFuncletPad());
+    // Check isSpecialTerminator().
+    EXPECT_EQ(LLVMI.isSpecialTerminator(), I.isSpecialTerminator());
+    // Check isOnlyUserOfAnyOperand().
+    EXPECT_EQ(LLVMI.isOnlyUserOfAnyOperand(), I.isOnlyUserOfAnyOperand());
+    // Check isLogicalShift().
+    EXPECT_EQ(LLVMI.isLogicalShift(), I.isLogicalShift());
+    // Check hasMetadata().
+    EXPECT_EQ(LLVMI.hasMetadata(), I.hasMetadata());
+    // Check hasMetadataOtherThanDebugLoc().
+    EXPECT_EQ(LLVMI.hasMetadataOtherThanDebugLoc(),
+              I.hasMetadataOtherThanDebugLoc());
     // Check isAssociative().
     EXPECT_EQ(LLVMI.isAssociative(), I.isAssociative());
     // Check isCommutative().
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index da5416395ec42..5823f4e14a854 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -11,7 +11,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Module.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/SandboxIR/TypesTest.cpp b/llvm/unittests/SandboxIR/TypesTest.cpp
index 40aa32fb08ed0..9bf02c97948eb 100644
--- a/llvm/unittests/SandboxIR/TypesTest.cpp
+++ b/llvm/unittests/SandboxIR/TypesTest.cpp
@@ -14,7 +14,8 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Module.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/SandboxIR/UtilsTest.cpp b/llvm/unittests/SandboxIR/UtilsTest.cpp
index ded3edf1206a4..033f217c3bcf7 100644
--- a/llvm/unittests/SandboxIR/UtilsTest.cpp
+++ b/llvm/unittests/SandboxIR/UtilsTest.cpp
@@ -7,13 +7,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/SandboxIR/Utils.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Module.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
@@ -54,3 +60,77 @@ define void @foo(ptr %arg0) {
   EXPECT_EQ(sandboxir::Utils::memoryLocationGetOrNone(Ld),
             MemoryLocation::getOrNone(LLVMLd));
 }
+
+TEST_F(UtilsTest, GetPointerDiffInBytes) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr) {
+  %gep0 = getelementptr inbounds float, ptr %ptr, i64 0
+  %gep1 = getelementptr inbounds float, ptr %ptr, i64 1
+  %gep2 = getelementptr inbounds float, ptr %ptr, i64 2
+  %gep3 = getelementptr inbounds float, ptr %ptr, i64 3
+
+  %ld0 = load float, ptr %gep0
+  %ld1 = load float, ptr %gep1
+  %ld2 = load float, ptr %gep2
+  %ld3 = load float, ptr %gep3
+
+  %v2ld0 = load <2 x float>, ptr %gep0
+  %v2ld1 = load <2 x float>, ptr %gep1
+  %v2ld2 = load <2 x float>, ptr %gep2
+  %v2ld3 = load <2 x float>, ptr %gep3
+
+  %v3ld0 = load <3 x float>, ptr %gep0
+  %v3ld1 = load <3 x float>, ptr %gep1
+  %v3ld2 = load <3 x float>, ptr %gep2
+  %v3ld3 = load <3 x float>, ptr %gep3
+  ret void
+}
+)IR");
+  llvm::Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII;
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  AssumptionCache AC(LLVMF);
+  BasicAAResult BAA(DL, LLVMF, TLI, AC, &DT);
+  AAResults AA(TLI);
+  AA.addAAResult(BAA);
+  LoopInfo LI(DT);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = *F.begin();
+  auto It = std::next(BB.begin(), 4);
+  auto *L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *L1 = cast<sandboxir::LoadInst>(&*It++);
+  auto *L2 = cast<sandboxir::LoadInst>(&*It++);
+  [[maybe_unused]] auto *L3 = cast<sandboxir::LoadInst>(&*It++);
+
+  auto *V2L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *V2L1 = cast<sandboxir::LoadInst>(&*It++);
+  auto *V2L2 = cast<sandboxir::LoadInst>(&*It++);
+  auto *V2L3 = cast<sandboxir::LoadInst>(&*It++);
+
+  [[maybe_unused]] auto *V3L0 = cast<sandboxir::LoadInst>(&*It++);
+  auto *V3L1 = cast<sandboxir::LoadInst>(&*It++);
+  [[maybe_unused]] auto *V3L2 = cast<sandboxir::LoadInst>(&*It++);
+  [[maybe_unused]] auto *V3L3 = cast<sandboxir::LoadInst>(&*It++);
+
+  // getPointerDiffInBytes
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L0, L1, SE, DL), 4);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L0, L2, SE, DL), 8);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L1, L0, SE, DL), -4);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L0, V2L0, SE, DL), 0);
+
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L0, V2L1, SE, DL), 4);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(L0, V3L1, SE, DL), 4);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(V2L0, V2L2, SE, DL), 8);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(V2L0, V2L3, SE, DL), 12);
+  EXPECT_EQ(*sandboxir::Utils::getPointerDiffInBytes(V2L3, V2L0, SE, DL), -12);
+
+  // atLowerAddress
+  EXPECT_TRUE(sandboxir::Utils::atLowerAddress(L0, L1, SE, DL));
+  EXPECT_FALSE(sandboxir::Utils::atLowerAddress(L1, L0, SE, DL));
+  EXPECT_FALSE(sandboxir::Utils::atLowerAddress(L3, V3L3, SE, DL));
+}
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
index 86b1d968094ca..9f1a3409c0c39 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/CMakeLists.txt
@@ -9,7 +9,6 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(SandboxVectorizerTests
   DependencyGraphTest.cpp
-  InstrIntervalTest.cpp
+  IntervalTest.cpp
   LegalityTest.cpp
-  RegionTest.cpp
   )
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
index f6bfd097f20a4..329d3617a31fa 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/DependencyGraphTest.cpp
@@ -8,7 +8,9 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/DependencyGraph.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/InstrIntervalTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
similarity index 59%
rename from llvm/unittests/Transforms/Vectorize/SandboxVectorizer/InstrIntervalTest.cpp
rename to llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
index e22bb78a07d30..0b2411151a965 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/InstrIntervalTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/IntervalTest.cpp
@@ -1,4 +1,4 @@
-//===- InstrIntervalTest.cpp ----------------------------------------------===//
+//===- IntervalTest.cpp ---------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,16 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Transforms/Vectorize/SandboxVectorizer/InstrInterval.h"
+#include "llvm/Transforms/Vectorize/SandboxVectorizer/Interval.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Constant.h"
+#include "llvm/SandboxIR/Context.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/SourceMgr.h"
-#include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
 
-struct InstrIntervalTest : public testing::Test {
+struct IntervalTest : public testing::Test {
   LLVMContext C;
   std::unique_ptr<Module> M;
 
@@ -27,7 +28,7 @@ struct InstrIntervalTest : public testing::Test {
   }
 };
 
-TEST_F(InstrIntervalTest, Basic) {
+TEST_F(IntervalTest, Basic) {
   parseIR(C, R"IR(
 define void @foo(i8 %v0) {
   %add0 = add i8 %v0, %v0
@@ -46,39 +47,40 @@ define void @foo(i8 %v0) {
   auto *I2 = &*It++;
   auto *Ret = &*It++;
 
-  sandboxir::InstrInterval Interval(I0, Ret);
+  sandboxir::Interval<sandboxir::Instruction> Intvl(I0, Ret);
 #ifndef NDEBUG
-  EXPECT_DEATH(sandboxir::InstrInterval(I1, I0), ".*before.*");
+  EXPECT_DEATH(sandboxir::Interval<sandboxir::Instruction>(I1, I0),
+               ".*before.*");
 #endif // NDEBUG
-  // Check InstrInterval(ArrayRef), from(), to().
+  // Check Interval<sandboxir::Instruction>(ArrayRef), from(), to().
   {
-    sandboxir::InstrInterval Interval(
+    sandboxir::Interval<sandboxir::Instruction> Intvl(
         SmallVector<sandboxir::Instruction *>({I0, Ret}));
-    EXPECT_EQ(Interval.top(), I0);
-    EXPECT_EQ(Interval.bottom(), Ret);
+    EXPECT_EQ(Intvl.top(), I0);
+    EXPECT_EQ(Intvl.bottom(), Ret);
   }
   {
-    sandboxir::InstrInterval Interval(
+    sandboxir::Interval<sandboxir::Instruction> Intvl(
         SmallVector<sandboxir::Instruction *>({Ret, I0}));
-    EXPECT_EQ(Interval.top(), I0);
-    EXPECT_EQ(Interval.bottom(), Ret);
+    EXPECT_EQ(Intvl.top(), I0);
+    EXPECT_EQ(Intvl.bottom(), Ret);
   }
   {
-    sandboxir::InstrInterval Interval(
+    sandboxir::Interval<sandboxir::Instruction> Intvl(
         SmallVector<sandboxir::Instruction *>({I0, I0}));
-    EXPECT_EQ(Interval.top(), I0);
-    EXPECT_EQ(Interval.bottom(), I0);
+    EXPECT_EQ(Intvl.top(), I0);
+    EXPECT_EQ(Intvl.bottom(), I0);
   }
 
   // Check empty().
-  EXPECT_FALSE(Interval.empty());
-  sandboxir::InstrInterval Empty;
+  EXPECT_FALSE(Intvl.empty());
+  sandboxir::Interval<sandboxir::Instruction> Empty;
   EXPECT_TRUE(Empty.empty());
-  sandboxir::InstrInterval One(I0, I0);
+  sandboxir::Interval<sandboxir::Instruction> One(I0, I0);
   EXPECT_FALSE(One.empty());
   // Check contains().
   for (auto &I : *BB) {
-    EXPECT_TRUE(Interval.contains(&I));
+    EXPECT_TRUE(Intvl.contains(&I));
     EXPECT_FALSE(Empty.contains(&I));
   }
   EXPECT_FALSE(One.contains(I1));
@@ -86,6 +88,6 @@ define void @foo(i8 %v0) {
   EXPECT_FALSE(One.contains(Ret));
   // Check iterator.
   auto BBIt = BB->begin();
-  for (auto &I : Interval)
+  for (auto &I : Intvl)
     EXPECT_EQ(&I, &*BBIt++);
 }
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
index a136be41ae363..89255a108ed6c 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/LegalityTest.cpp
@@ -8,7 +8,7 @@
 
 #include "llvm/Transforms/Vectorize/SandboxVectorizer/Legality.h"
 #include "llvm/AsmParser/Parser.h"
-#include "llvm/SandboxIR/SandboxIR.h"
+#include "llvm/SandboxIR/Instruction.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index a7ff51067c8d0..a5d44e9a57246 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1230,9 +1230,9 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
     VPValue Op2;
     VPInstruction VPInst(Instruction::Add, {&Op1, &Op2});
     VPRecipeBase &Recipe = VPInst;
-    EXPECT_TRUE(Recipe.mayHaveSideEffects());
+    EXPECT_FALSE(Recipe.mayHaveSideEffects());
     EXPECT_TRUE(Recipe.mayReadFromMemory());
-    EXPECT_TRUE(Recipe.mayWriteToMemory());
+    EXPECT_FALSE(Recipe.mayWriteToMemory());
     EXPECT_TRUE(Recipe.mayReadOrWriteMemory());
   }
   {
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 8876bb3ad31e1..1b95082ad0715 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -131,9 +131,9 @@ void CallingConvEmitter::EmitAction(const Record *Action, unsigned Indent,
     O << IndentStr << "if (";
 
     if (Action->isSubClassOf("CCIfType")) {
-      ListInit *VTs = Action->getValueAsListInit("VTs");
+      const ListInit *VTs = Action->getValueAsListInit("VTs");
       for (unsigned i = 0, e = VTs->size(); i != e; ++i) {
-        Record *VT = VTs->getElementAsRecord(i);
+        const Record *VT = VTs->getElementAsRecord(i);
         if (i != 0)
           O << " ||\n    " << IndentStr;
         O << "LocVT == " << getEnumName(getValueType(VT));
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index a4b5495092867..bab53adbaefe3 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -39,10 +39,10 @@ struct DXILOperationDesc {
   StringRef OpClass;  // name of the opcode class
   StringRef Doc;      // the documentation description of this instruction
   // Vector of operand type records - return type is at index 0
-  SmallVector<Record *> OpTypes;
-  SmallVector<Record *> OverloadRecs;
-  SmallVector<Record *> StageRecs;
-  SmallVector<Record *> AttrRecs;
+  SmallVector<const Record *> OpTypes;
+  SmallVector<const Record *> OverloadRecs;
+  SmallVector<const Record *> StageRecs;
+  SmallVector<const Record *> AttrRecs;
   StringRef Intrinsic; // The llvm intrinsic map to OpName. Default is "" which
                        // means no map exists
   SmallVector<StringRef, 4>
@@ -57,8 +57,8 @@ struct DXILOperationDesc {
 /// In-place sort TableGen records of class with a field
 ///    Version dxil_version
 /// in the ascending version order.
-static void AscendingSortByVersion(std::vector<Record *> &Recs) {
-  std::sort(Recs.begin(), Recs.end(), [](Record *RecA, Record *RecB) {
+static void AscendingSortByVersion(std::vector<const Record *> &Recs) {
+  sort(Recs, [](const Record *RecA, const Record *RecB) {
     unsigned RecAMaj =
         RecA->getValueAsDef("dxil_version")->getValueAsInt("Major");
     unsigned RecAMin =
@@ -82,13 +82,12 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   OpCode = R->getValueAsInt("OpCode");
 
   Doc = R->getValueAsString("Doc");
-  SmallVector<Record *> ParamTypeRecs;
+  SmallVector<const Record *> ParamTypeRecs;
 
   ParamTypeRecs.push_back(R->getValueAsDef("result"));
 
-  std::vector<Record *> ArgTys = R->getValueAsListOfDefs("arguments");
-  for (auto Ty : ArgTys) {
-    ParamTypeRecs.push_back(Ty);
+  for (const Record *ArgTy : R->getValueAsListOfDefs("arguments")) {
+    ParamTypeRecs.push_back(ArgTy);
   }
   size_t ParamTypeRecsSize = ParamTypeRecs.size();
   // Populate OpTypes with return type and parameter types
@@ -100,7 +99,7 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   // llvm/IR/Intrinsics.td
   OverloadParamIndex = -1; // A sigil meaning none.
   for (unsigned i = 0; i < ParamTypeRecsSize; i++) {
-    Record *TR = ParamTypeRecs[i];
+    const Record *TR = ParamTypeRecs[i];
     // Track operation parameter indices of any overload types
     if (TR->getValueAsInt("isOverload")) {
       if (OverloadParamIndex != -1) {
@@ -117,17 +116,17 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 
   // Get overload records
-  std::vector<Record *> Recs = R->getValueAsListOfDefs("overloads");
+  std::vector<const Record *> Recs = R->getValueAsListOfConstDefs("overloads");
 
   // Sort records in ascending order of DXIL version
   AscendingSortByVersion(Recs);
 
-  for (Record *CR : Recs) {
+  for (const Record *CR : Recs) {
     OverloadRecs.push_back(CR);
   }
 
   // Get stage records
-  Recs = R->getValueAsListOfDefs("stages");
+  Recs = R->getValueAsListOfConstDefs("stages");
 
   if (Recs.empty()) {
     PrintFatalError(R, Twine("Atleast one specification of valid stage for ") +
@@ -137,17 +136,17 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   // Sort records in ascending order of DXIL version
   AscendingSortByVersion(Recs);
 
-  for (Record *CR : Recs) {
+  for (const Record *CR : Recs) {
     StageRecs.push_back(CR);
   }
 
   // Get attribute records
-  Recs = R->getValueAsListOfDefs("attributes");
+  Recs = R->getValueAsListOfConstDefs("attributes");
 
   // Sort records in ascending order of DXIL version
   AscendingSortByVersion(Recs);
 
-  for (Record *CR : Recs) {
+  for (const Record *CR : Recs) {
     AttrRecs.push_back(CR);
   }
 
@@ -201,7 +200,7 @@ static StringRef getOverloadKindStr(const Record *R) {
 /// \return std::string string representation of overload mask string
 ///         predicated by DXIL Version. E.g.,
 //          {{{1, 0}, Mask1}, {{1, 2}, Mask2}, ...}
-static std::string getOverloadMaskString(const SmallVector<Record *> Recs) {
+static std::string getOverloadMaskString(ArrayRef<const Record *> Recs) {
   std::string MaskString = "";
   std::string Prefix = "";
   MaskString.append("{");
@@ -247,7 +246,7 @@ static std::string getOverloadMaskString(const SmallVector<Record *> Recs) {
 /// \return std::string string representation of stages mask string
 ///         predicated by DXIL Version. E.g.,
 //          {{{1, 0}, Mask1}, {{1, 2}, Mask2}, ...}
-static std::string getStageMaskString(const SmallVector<Record *> Recs) {
+static std::string getStageMaskString(ArrayRef<const Record *> Recs) {
   std::string MaskString = "";
   std::string Prefix = "";
   MaskString.append("{");
@@ -290,7 +289,7 @@ static std::string getStageMaskString(const SmallVector<Record *> Recs) {
 /// \return std::string string representation of stages mask string
 ///         predicated by DXIL Version. E.g.,
 //          {{{1, 0}, Mask1}, {{1, 2}, Mask2}, ...}
-static std::string getAttributeMaskString(const SmallVector<Record *> Recs) {
+static std::string getAttributeMaskString(ArrayRef<const Record *> Recs) {
   std::string MaskString = "";
   std::string Prefix = "";
   MaskString.append("{");
diff --git a/llvm/utils/TableGen/DirectiveEmitter.cpp b/llvm/utils/TableGen/DirectiveEmitter.cpp
index fafdfa0db89af..dc9ed2b08972e 100644
--- a/llvm/utils/TableGen/DirectiveEmitter.cpp
+++ b/llvm/utils/TableGen/DirectiveEmitter.cpp
@@ -343,8 +343,9 @@ static void GenerateGetKindClauseVal(const DirectiveLanguage &DirLang,
     if (ClauseVals.size() <= 0)
       continue;
 
-    auto DefaultIt = find_if(
-        ClauseVals, [](Record *CV) { return CV->getValueAsBit("isDefault"); });
+    auto DefaultIt = find_if(ClauseVals, [](const Record *CV) {
+      return CV->getValueAsBit("isDefault");
+    });
 
     if (DefaultIt == ClauseVals.end()) {
       PrintError("At least one val in Clause " + C.getFormattedName() +
@@ -505,7 +506,7 @@ static void EmitLeafTable(const DirectiveLanguage &DirLang, raw_ostream &OS,
   std::vector<LeafList> LeafTable(Directives.size());
   for (auto [Idx, Rec] : enumerate(Directives)) {
     Directive Dir(Rec);
-    std::vector<Record *> Leaves = Dir.getLeafConstructs();
+    std::vector<const Record *> Leaves = Dir.getLeafConstructs();
 
     auto &List = LeafTable[Idx];
     List.resize(MaxLeafCount + 2);
@@ -680,7 +681,7 @@ static void GenerateGetDirectiveAssociation(const DirectiveLanguage &DirLang,
       return AS;
     }
     // Compute the association from leaf constructs.
-    std::vector<Record *> leaves = D.getLeafConstructs();
+    std::vector<const Record *> leaves = D.getLeafConstructs();
     if (leaves.empty()) {
       errs() << D.getName() << '\n';
       PrintFatalError(errorPrefixFor(D) +
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 46605095ba85f..5653434ddd682 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -1357,7 +1357,7 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
   OS << "#endif // GET_INSTRINFO_SCHED_ENUM\n\n";
 }
 
-static void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) {
+static void EmitInstrInfo(const RecordKeeper &RK, raw_ostream &OS) {
   RK.startTimer("Analyze DAG patterns");
   InstrInfoEmitter(RK).run(OS);
   RK.startTimer("Emit map table");
diff --git a/llvm/utils/TableGen/OptionParserEmitter.cpp b/llvm/utils/TableGen/OptionParserEmitter.cpp
index 5ae6f773a3c60..424cf16e719d5 100644
--- a/llvm/utils/TableGen/OptionParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptionParserEmitter.cpp
@@ -471,13 +471,13 @@ static void EmitOptionParser(const RecordKeeper &Records, raw_ostream &OS) {
 
     std::vector<std::pair<std::vector<std::string>, StringRef>>
         HelpTextsForVariants;
-    for (Record *VisibilityHelp :
+    for (const Record *VisibilityHelp :
          R.getValueAsListOfDefs("HelpTextsForVariants")) {
       ArrayRef<Init *> Visibilities =
           VisibilityHelp->getValueAsListInit("Visibilities")->getValues();
 
       std::vector<std::string> VisibilityNames;
-      for (Init *Visibility : Visibilities)
+      for (const Init *Visibility : Visibilities)
         VisibilityNames.push_back(Visibility->getAsUnquotedString());
 
       HelpTextsForVariants.push_back(std::make_pair(
diff --git a/llvm/utils/TableGen/OptionRSTEmitter.cpp b/llvm/utils/TableGen/OptionRSTEmitter.cpp
index b798896a80963..1b4c4cad4f0a4 100644
--- a/llvm/utils/TableGen/OptionRSTEmitter.cpp
+++ b/llvm/utils/TableGen/OptionRSTEmitter.cpp
@@ -18,7 +18,6 @@ using namespace llvm;
 /// and emits a RST man page.
 static void EmitOptionRST(const RecordKeeper &Records, raw_ostream &OS) {
   llvm::StringMap<std::vector<const Record *>> OptionsByGroup;
-  std::vector<Record *> OptionsWithoutGroup;
 
   // Get the options.
   std::vector<const Record *> Opts = Records.getAllDerivedDefinitions("Option");
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index dfa10f74974c7..8ab7bdcd2214b 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -475,7 +475,7 @@ void X86FoldTablesEmitter::addEntryWithFlags(FoldTable &Table,
   StringRef RegInstName = RegRec->getName();
   unsigned DropLen =
       RegInstName.ends_with("rkz") ? 2 : (RegInstName.ends_with("rk") ? 1 : 0);
-  Record *BaseDef =
+  const Record *BaseDef =
       DropLen ? Records.getDef(RegInstName.drop_back(DropLen)) : nullptr;
   bool IsMoveReg =
       BaseDef ? Target.getInstruction(BaseDef).isMoveReg : RegInst->isMoveReg;
diff --git a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
index f68c727cbe923..47df5bf0df8e5 100644
--- a/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
+++ b/llvm/utils/TableGen/X86InstrMappingEmitter.cpp
@@ -215,7 +215,7 @@ void X86InstrMappingEmitter::emitCompressEVEXTable(
     StringRef Name = Rec->getName();
     const CodeGenInstruction *NewInst = nullptr;
     if (ManualMap.find(Name) != ManualMap.end()) {
-      Record *NewRec = Records.getDef(ManualMap.at(Rec->getName()));
+      const Record *NewRec = Records.getDef(ManualMap.at(Rec->getName()));
       assert(NewRec && "Instruction not found!");
       NewInst = &Target.getInstruction(NewRec);
     } else if (Name.ends_with("_EVEX")) {
diff --git a/llvm/utils/TableGen/X86ManualInstrMapping.def b/llvm/utils/TableGen/X86ManualInstrMapping.def
index bc539d792f38d..7c5a6033237fe 100644
--- a/llvm/utils/TableGen/X86ManualInstrMapping.def
+++ b/llvm/utils/TableGen/X86ManualInstrMapping.def
@@ -246,14 +246,14 @@ ENTRY(VCVTTPD2DQZ256rm, VCVTTPD2DQYrm)
 ENTRY(VCVTTPD2DQZ256rr, VCVTTPD2DQYrr)
 ENTRY(VDIVPDZ256rm, VDIVPDYrm)
 ENTRY(VDIVPDZ256rr, VDIVPDYrr)
-ENTRY(VEXTRACTF64x2Z256mr, VEXTRACTF128mri)
-ENTRY(VEXTRACTF64x2Z256rr, VEXTRACTF128rri)
-ENTRY(VEXTRACTI64x2Z256mr, VEXTRACTI128mri)
-ENTRY(VEXTRACTI64x2Z256rr, VEXTRACTI128rri)
-ENTRY(VINSERTF64x2Z256rm, VINSERTF128rmi)
-ENTRY(VINSERTF64x2Z256rr, VINSERTF128rri)
-ENTRY(VINSERTI64x2Z256rm, VINSERTI128rmi)
-ENTRY(VINSERTI64x2Z256rr, VINSERTI128rri)
+ENTRY(VEXTRACTF64x2Z256mri, VEXTRACTF128mri)
+ENTRY(VEXTRACTF64x2Z256rri, VEXTRACTF128rri)
+ENTRY(VEXTRACTI64x2Z256mri, VEXTRACTI128mri)
+ENTRY(VEXTRACTI64x2Z256rri, VEXTRACTI128rri)
+ENTRY(VINSERTF64x2Z256rmi, VINSERTF128rmi)
+ENTRY(VINSERTF64x2Z256rri, VINSERTF128rri)
+ENTRY(VINSERTI64x2Z256rmi, VINSERTI128rmi)
+ENTRY(VINSERTI64x2Z256rri, VINSERTI128rri)
 ENTRY(VMAXCPDZ256rm, VMAXCPDYrm)
 ENTRY(VMAXCPDZ256rr, VMAXCPDYrr)
 ENTRY(VMAXPDZ256rm, VMAXPDYrm)
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
index 3b640ae41b9f6..7a6c360e88c14 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -141,7 +141,7 @@ static_library("Checkers") {
     "VforkChecker.cpp",
     "VirtualCallChecker.cpp",
     "WebKit/ASTUtils.cpp",
-    "WebKit/NoUncountedMembersChecker.cpp",
+    "WebKit/RawPtrRefMemberChecker.cpp",
     "WebKit/PtrTypesSemantics.cpp",
     "WebKit/RefCntblBaseVirtualDtorChecker.cpp",
     "WebKit/UncountedCallArgsChecker.cpp",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 91d547da201f2..3e165c3d6f634 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -95,7 +95,6 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/find_if.h",
       "__algorithm/find_if_not.h",
       "__algorithm/find_segment_if.h",
-      "__algorithm/fold.h",
       "__algorithm/for_each.h",
       "__algorithm/for_each_n.h",
       "__algorithm/for_each_segment.h",
@@ -170,6 +169,7 @@ if (current_toolchain == default_toolchain) {
       "__algorithm/ranges_find_if.h",
       "__algorithm/ranges_find_if_not.h",
       "__algorithm/ranges_find_last.h",
+      "__algorithm/ranges_fold.h",
       "__algorithm/ranges_for_each.h",
       "__algorithm/ranges_for_each_n.h",
       "__algorithm/ranges_generate.h",
@@ -608,6 +608,7 @@ if (current_toolchain == default_toolchain) {
       "__memory/allocator_arg_t.h",
       "__memory/allocator_destructor.h",
       "__memory/allocator_traits.h",
+      "__memory/array_cookie.h",
       "__memory/assume_aligned.h",
       "__memory/auto_ptr.h",
       "__memory/builtin_new_allocator.h",
diff --git a/llvm/utils/gn/secondary/llvm/lib/SandboxIR/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/SandboxIR/BUILD.gn
index 529412f77cc29..51aae0241499b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/SandboxIR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/SandboxIR/BUILD.gn
@@ -6,10 +6,19 @@ static_library("SandboxIR") {
     "//llvm/lib/Support",
   ]
   sources = [
+    "Argument.cpp",
+    "BasicBlock.cpp",
+    "Constant.cpp",
+    "Context.cpp",
+    "Instruction.cpp",
+    "Module.cpp",
     "Pass.cpp",
     "PassManager.cpp",
+    "Region.cpp",
     "SandboxIR.cpp",
     "Tracker.cpp",
     "Type.cpp",
+    "User.cpp",
+    "Value.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 66db9100fb597..9f85f2ec59511 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -15,7 +15,6 @@ static_library("Vectorize") {
     "SLPVectorizer.cpp",
     "SandboxVectorizer/DependencyGraph.cpp",
     "SandboxVectorizer/Passes/BottomUpVec.cpp",
-    "SandboxVectorizer/Region.cpp",
     "SandboxVectorizer/SandboxVectorizer.cpp",
     "VPlan.cpp",
     "VPlanAnalysis.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
index 5828d122aede9..11045f4315877 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/SandboxIR/BUILD.gn
@@ -9,6 +9,7 @@ unittest("SandboxIRTests") {
   ]
   sources = [
     "PassTest.cpp",
+    "RegionTest.cpp",
     "SandboxIRTest.cpp",
     "TrackerTest.cpp",
     "TypesTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
index a9e2170ce4be7..a01525a0c80b6 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/BUILD.gn
@@ -11,8 +11,7 @@ unittest("SandboxVectorizerTests") {
   ]
   sources = [
     "DependencyGraphTest.cpp",
-    "InstrIntervalTest.cpp",
+    "IntervalTest.cpp",
     "LegalityTest.cpp",
-    "RegionTest.cpp",
   ]
 }
diff --git a/mlir/include/mlir/Dialect/Arith/Utils/Utils.h b/mlir/include/mlir/Dialect/Arith/Utils/Utils.h
index 76f5825025739..d759299cbf762 100644
--- a/mlir/include/mlir/Dialect/Arith/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Arith/Utils/Utils.h
@@ -130,6 +130,10 @@ namespace arith {
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values);
 Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
                     Type resultType);
+
+// Map strings to float types.
+std::optional<FloatType> parseFloatType(MLIRContext *ctx, StringRef name);
+
 } // namespace arith
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 8584a25f8b3d6..030160821bd82 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -551,7 +551,15 @@ def LLVM_InvokeOp : LLVM_Op<"invoke", [
                    Variadic<LLVM_Type>:$normalDestOperands,
                    Variadic<LLVM_Type>:$unwindDestOperands,
                    OptionalAttr<DenseI32ArrayAttr>:$branch_weights,
-                   DefaultValuedAttr<CConv, "CConv::C">:$CConv);
+                   DefaultValuedAttr<CConv, "CConv::C">:$CConv,
+                   VariadicOfVariadic<LLVM_Type,
+                                      "op_bundle_sizes">:$op_bundle_operands,
+                   DenseI32ArrayAttr:$op_bundle_sizes,
+                   DefaultValuedProperty<
+                     ArrayProperty<StringProperty, "operand bundle tags">,
+                     "ArrayRef<std::string>{}",
+                     "SmallVector<std::string>{}"
+                   >:$op_bundle_tags);
   let results = (outs Optional<LLVM_Type>:$result);
   let successors = (successor AnySuccessor:$normalDest,
                               AnySuccessor:$unwindDest);
@@ -607,7 +615,8 @@ def LLVM_VaArgOp : LLVM_Op<"va_arg"> {
 //===----------------------------------------------------------------------===//
 
 def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
-                    [DeclareOpInterfaceMethods<FastmathFlagsInterface>,
+                    [AttrSizedOperandSegments,
+                     DeclareOpInterfaceMethods<FastmathFlagsInterface>,
                      DeclareOpInterfaceMethods<CallOpInterface>,
                      DeclareOpInterfaceMethods<SymbolUserOpInterface>,
                      DeclareOpInterfaceMethods<BranchWeightOpInterface>]> {
@@ -661,8 +670,15 @@ def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
                   OptionalAttr<LLVM_MemoryEffectsAttr>:$memory_effects,
                   OptionalAttr<UnitAttr>:$convergent,
                   OptionalAttr<UnitAttr>:$no_unwind,
-                  OptionalAttr<UnitAttr>:$will_return
-                  );
+                  OptionalAttr<UnitAttr>:$will_return,
+                  VariadicOfVariadic<LLVM_Type,
+                                     "op_bundle_sizes">:$op_bundle_operands,
+                  DenseI32ArrayAttr:$op_bundle_sizes,
+                  DefaultValuedProperty<
+                    ArrayProperty<StringProperty, "operand bundle tags">,
+                    "ArrayRef<std::string>{}",
+                    "SmallVector<std::string>{}"
+                  >:$op_bundle_tags);
   // Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
   let arguments = !con(args, aliasAttrs);
   let results = (outs Optional<LLVM_Type>:$result);
@@ -682,6 +698,7 @@ def LLVM_CallOp : LLVM_MemAccessOpBase<"call",
     OpBuilder<(ins "LLVMFunctionType":$calleeType, "StringRef":$callee,
                    CArg<"ValueRange", "{}">:$args)>
   ];
+  let hasVerifier = 1;
   let hasCustomAssemblyFormat = 1;
   let extraClassDeclaration = [{
     /// Returns the callee function type.
@@ -1895,7 +1912,8 @@ def LLVM_InlineAsmOp : LLVM_Op<"inline_asm", [DeclareOpInterfaceMethods<MemoryEf
 
 def LLVM_CallIntrinsicOp
     : LLVM_Op<"call_intrinsic",
-              [DeclareOpInterfaceMethods<FastmathFlagsInterface>]> {
+              [AttrSizedOperandSegments,
+               DeclareOpInterfaceMethods<FastmathFlagsInterface>]> {
   let summary = "Call to an LLVM intrinsic function.";
   let description = [{
     Call the specified llvm intrinsic. If the intrinsic is overloaded, use
@@ -1903,13 +1921,25 @@ def LLVM_CallIntrinsicOp
   }];
   let arguments = (ins StrAttr:$intrin, Variadic<LLVM_Type>:$args,
                        DefaultValuedAttr<LLVM_FastmathFlagsAttr,
-                                         "{}">:$fastmathFlags);
+                                         "{}">:$fastmathFlags,
+                       VariadicOfVariadic<LLVM_Type,
+                                          "op_bundle_sizes">:$op_bundle_operands,
+                       DenseI32ArrayAttr:$op_bundle_sizes,
+                       DefaultValuedProperty<
+                         ArrayProperty<StringProperty, "operand bundle tags">,
+                         "ArrayRef<std::string>{}",
+                         "SmallVector<std::string>{}"
+                       >:$op_bundle_tags);
   let results = (outs Optional<LLVM_Type>:$results);
   let llvmBuilder = [{
     return convertCallLLVMIntrinsicOp(op, builder, moduleTranslation);
   }];
   let assemblyFormat = [{
-    $intrin `(` $args `)` `:` functional-type($args, $results) attr-dict
+    $intrin `(` $args `)`
+    ( custom<OpBundles>($op_bundle_operands, type($op_bundle_operands),
+                        $op_bundle_tags)^ )?
+    `:` functional-type($args, $results)
+    attr-dict
   }];
 
   let hasVerifier = 1;
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
index db25c9b241734..9ede21e87cf53 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.h
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Transform/IR/TransformAttrs.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/IR/OpImplementation.h"
@@ -42,10 +43,6 @@ class UnPackOp;
 } // namespace tensor
 
 namespace transform {
-class AnyOpType;
-class AnyValueType;
-class OperationType;
-class TransformHandleTypeInterface;
 // Types needed for builders.
 struct TileSizesSpec {};
 struct NumThreadsSpec {};
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 0208f854f799e..48e657cca96e3 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -549,7 +549,7 @@ namespace detail {
 struct PackingResult {
   SmallVector<OpFoldResult> offsets, sizes, strides;
   SmallVector<Value> clonedLoopIvs, leadingPackedTensorIndexings;
-  GenericOp maybeTransposeOp;
+  TransposeOp maybeTransposeOp;
   tensor::PadOp hoistedPadOp;
 };
 
@@ -568,9 +568,9 @@ buildPackingLoopNest(RewriterBase &rewriter, tensor::PadOp opToHoist,
 /// a larger tensor. On success, `opToHoist` is replaced by the cloned version
 /// in the packing loop so the caller can continue reasoning about the padding
 /// operation. If `transposeVector` is non-empty, hoist padding introduces a
-/// GenericOp to transpose the padded tensor before inserting it into the packed
-/// tensor. A `transposeVector` can change the storage order of the padded
-/// tensor but does not change the order of the pack or compute loops.
+/// TransposeOp to transpose the padded tensor before inserting it into the
+/// packed tensor. A `transposeVector` can change the storage order of the
+/// padded tensor but does not change the order of the pack or compute loops.
 ///
 /// TODO: In the future, we should consider rewriting as a tensor.pack after
 /// hoisting since this abstraction is now available.
@@ -615,13 +615,13 @@ FailureOr<Value>
 hoistPaddingOnTensors(RewriterBase &rewriter, tensor::PadOp opToHoist,
                       int64_t numLoops, ArrayRef<int64_t> transposeVector,
                       tensor::PadOp &hoistedOp,
-                      SmallVectorImpl<GenericOp> &transposeOps);
+                      SmallVectorImpl<TransposeOp> &transposeOps);
 /// Calls into `hoistPaddingOnTensors` with a local IRRewriter.
 FailureOr<Value>
 hoistPaddingOnTensors(tensor::PadOp opToHoist, int64_t numLoops,
                       ArrayRef<int64_t> transposeVector,
                       tensor::PadOp &hoistedOp,
-                      SmallVectorImpl<GenericOp> &transposeOps);
+                      SmallVectorImpl<TransposeOp> &transposeOps);
 
 /// Apply padding and hoisting to `linalgOp` according to the configuration
 /// specified in `options`.
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index f1df49ce3eaa3..1e4f3004dec7e 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -75,12 +75,6 @@ bool isReductionIterator(utils::IteratorType iteratorType);
 Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
                             Value source, Value pad, bool nofold);
 
-/// Returns a GenericOp that transposes `inputTensor` into `outputTensor`
-/// using `transposeVector` to permute the `inputTensor` dimensions.
-GenericOp makeTransposeOp(OpBuilder &b, Location loc, Value inputTensor,
-                          Value outputTensor,
-                          ArrayRef<int64_t> transposeVector);
-
 /// Returns GenericOp that copies an n-D memref. Unlike the current
 /// implementation of memref::CopyOp, this op can further tile, lower to loops
 /// or vectorize.
diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
index 2dd7f6431f03e..2974bb344ad96 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.h
@@ -56,11 +56,13 @@ void populateMathPolynomialApproximationPatterns(
 void populateUpliftToFMAPatterns(RewritePatternSet &patterns);
 
 namespace math {
-void populateLegalizeToF32TypeConverter(TypeConverter &typeConverter);
-void populateLegalizeToF32ConversionTarget(ConversionTarget &target,
-                                           TypeConverter &typeConverter);
-void populateLegalizeToF32Patterns(RewritePatternSet &patterns,
-                                   TypeConverter &typeConverter);
+void populateExtendToSupportedTypesTypeConverter(
+    TypeConverter &typeConverter, const SetVector<Type> &sourceTypes,
+    Type targetType);
+void populateExtendToSupportedTypesConversionTarget(
+    ConversionTarget &target, TypeConverter &typeConverter);
+void populateExtendToSupportedTypesPatterns(RewritePatternSet &patterns,
+                                            TypeConverter &typeConverter);
 } // namespace math
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Math/Transforms/Passes.td b/mlir/include/mlir/Dialect/Math/Transforms/Passes.td
index e870e714bfda5..a84c89020d4f3 100644
--- a/mlir/include/mlir/Dialect/Math/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Math/Transforms/Passes.td
@@ -19,7 +19,7 @@ def MathUpliftToFMA : Pass<"math-uplift-to-fma"> {
   let dependentDialects = ["math::MathDialect"];
 }
 
-def MathLegalizeToF32 : Pass<"math-legalize-to-f32"> {
+def MathExtendToSupportedTypes : Pass<"math-extend-to-supported-types"> {
   let summary = "Legalize floating-point math ops on low-precision floats";
   let description = [{
     On many targets, the math functions are not implemented for floating-point
@@ -28,11 +28,19 @@ def MathLegalizeToF32 : Pass<"math-legalize-to-f32"> {
 
     This pass explicitly legalizes these math functions by inserting
     `arith.extf` and `arith.truncf` pairs around said op, which preserves
-    the original semantics while enabling lowering.
+    the original semantics while enabling lowering. The extra supported floating-point
+    types for the target are passed as arguments. Types f64 and f32 are implicitly 
+    supported.
 
     As an exception, this pass does not legalize `math.fma`, because
     that is an operation frequently implemented at low precisions.
   }];
+  let options = [
+    ListOption<"extraTypeStrs", "extra-types", "std::string",
+      "MLIR types with arithmetic support on a given target (f64 and f32 are implicitly supported)">,
+    Option<"targetTypeStr", "target-type", "std::string", "\"f32\"",
+      "MLIR type to convert the unsupported source types to">,
+  ];
   let dependentDialects = ["math::MathDialect", "arith::ArithDialect"];
 }
 
diff --git a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
index c08f399ee182d..9f57627c321fb 100644
--- a/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Quant/CMakeLists.txt
@@ -1,6 +1,2 @@
-add_mlir_dialect(QuantOps quant)
-add_mlir_doc(QuantOps QuantDialect Dialects/ -gen-dialect-doc)
-
-set(LLVM_TARGET_DEFINITIONS QuantDialectBytecode.td)
-mlir_tablegen(QuantDialectBytecode.cpp.inc -gen-bytecode -bytecode-dialect="Quant")
-add_public_tablegen_target(MLIRQuantDialectBytecodeIncGen)
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Dialect/Quant/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Quant/IR/CMakeLists.txt
new file mode 100644
index 0000000000000..c08f399ee182d
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/IR/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_mlir_dialect(QuantOps quant)
+add_mlir_doc(QuantOps QuantDialect Dialects/ -gen-dialect-doc)
+
+set(LLVM_TARGET_DEFINITIONS QuantDialectBytecode.td)
+mlir_tablegen(QuantDialectBytecode.cpp.inc -gen-bytecode -bytecode-dialect="Quant")
+add_public_tablegen_target(MLIRQuantDialectBytecodeIncGen)
diff --git a/mlir/include/mlir/Dialect/Quant/QuantOps.h b/mlir/include/mlir/Dialect/Quant/IR/Quant.h
similarity index 59%
rename from mlir/include/mlir/Dialect/Quant/QuantOps.h
rename to mlir/include/mlir/Dialect/Quant/IR/Quant.h
index 14fb3035ab0d3..11a969a3ee519 100644
--- a/mlir/include/mlir/Dialect/Quant/QuantOps.h
+++ b/mlir/include/mlir/Dialect/Quant/IR/Quant.h
@@ -1,4 +1,4 @@
-//===- QuantOps.h - Quantization Ops and Types ------------------*- C++ -*-===//
+//===- Quant.h - Quantization Ops -------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_QUANT_QUANTOPS_H_
-#define MLIR_DIALECT_QUANT_QUANTOPS_H_
+#ifndef MLIR_DIALECT_QUANT_IR_QUANT_H_
+#define MLIR_DIALECT_QUANT_IR_QUANT_H_
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
@@ -19,9 +19,19 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Support/MathExtras.h"
 
-#include "mlir/Dialect/Quant/QuantOpsDialect.h.inc"
+#include "mlir/Dialect/Quant/IR/QuantOpsDialect.h.inc"
+
+namespace mlir {
+namespace quant {
+
+class QuantizedType;
+class UniformQuantizedType;
+class UniformQuantizedPerAxisType;
+
+} // namespace quant
+} // namespace mlir
 
 #define GET_OP_CLASSES
-#include "mlir/Dialect/Quant/QuantOps.h.inc"
+#include "mlir/Dialect/Quant/IR/QuantOps.h.inc"
 
-#endif // MLIR_DIALECT_QUANT_QUANTOPS_H_
+#endif // MLIR_DIALECT_QUANT_IR_QUANT_H_
diff --git a/mlir/include/mlir/Dialect/Quant/IR/QuantBase.td b/mlir/include/mlir/Dialect/Quant/IR/QuantBase.td
new file mode 100644
index 0000000000000..791cb9de48d05
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/IR/QuantBase.td
@@ -0,0 +1,297 @@
+//===- QuantBase.td - Quantization dialect base ------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Quantization dialect, types, and traits.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef QUANT_BASE
+#define QUANT_BASE
+
+include "mlir/IR/OpBase.td"
+
+def Quant_Dialect : Dialect {
+  let name = "quant";
+  let description = [{
+    The `quant` dialect offers a framework for defining and manipulating
+    quantized values. Central to this framework is the `!quant.uniform` data
+    type, used to represent quantized values. This dialect also provides a
+    suite of operations to handle and convert quantized values between their
+    original floating-point representations and the optimized, lower bit-width
+    integer representations. The `quant` dialect is instrumented with
+    transformation passes to lower these operations into other core MLIR
+    dialects, while also flattening all occurrences of quantized types into
+    their integer counterparts.
+
+
+    ## The `!quant.uniform` type
+
+    The quantization process establishes a relationship between two types of
+    values: an *expressed value* and a *stored value*. The former refers to the
+    floating-point representation used in an original machine learning model,
+    capturing the precise numerical characteristics needed for accurate
+    calculations. The latter is the simplified integer representation that
+    resides in memory after quantization. The `!quant.uniform` data type
+    encodes the necessary information for (lossy) round-trip conversion between
+    an expressed and a stored value.
+
+    The `quant.uniform` type has two variants: per-layer quantization and
+    per-channel (or per-axis) quantization. In per-layer quantization, the
+    quantization information affects an entire tensor uniformly. Conversely, in
+    per-channel quantization, the data type encodes the specific tensor axis
+    that serves as the channel and includes quantization information for each
+    individual channel within the tensor. Below are the specific syntactic and
+    semantic considerations for each modality.
+
+
+    ### Per-layer quantization
+
+    This is the general syntax of the `!quant.uniform` type representing
+    per-layer quantization:
+
+    ```
+    `!quant.uniform` `<`
+      storedType (`<` storageMin `:` storageMax `>`)? `:`
+      expressedType `,`
+      scale (`:` zeroPoint)?
+    `>`
+    ```
+
+    The type contains the following parameters:
+
+    - `storedType`: Integer type of the value stored in memory. This type
+      conveys the bit width and signedness of the quantized stored value.
+      Signed integer types are represented as `'i' bitWidth` (e.g., `i8`),
+      while unsigned integer types are represented as `'u' bitWidth` (e.g.,
+      `u8`).
+
+    - `storageMin`, `storageMax`: Optional bounds for the stored value. If
+      given, they must be within the range of `storedType`. If omitted, the
+      entire range of `storedType` is allowed (e.g., `-128...127` for `i8` or
+      `0...255` for `u8`).
+
+    - `expressedType`: Floating-point type of the value expressed by this
+      quantized type (e.g., `f32`, `f80`, `bf16`, or `tf32`).
+
+    - `scale`: Floating-point value of type `expressedType` used in the
+      conversion between stored and expressed values.
+
+    - `zeroPoint`: Optional integer value of type `storageType` used in the
+      conversion between stored and expressed values. If omitted, the default
+      is 0.
+
+    Type conversions, rounding methods, and clamping actions aside, the
+    relationship between the expressed and stored values as encoded in a
+    quantized type is denoted by the following formula:
+
+    $$
+    expressedValue = (storedValue ~-~ zeroPoint) ~\times~ scale
+    $$
+
+    Operations `quant.qcast` (quantize cast) and `quant.dcast` (dequantize
+    cast) can be used to quantize a floating-point value and dequantize a
+    stored value, respectively. See the documentation for these operations for
+    details on how the quantization and dequantization processes are influenced
+    by the `!quant.uniform` type parameters.
+
+    Here are some examples of the use of `!quant.uniform` with per-layer
+    quantization:
+
+    ```
+    // An 8-bit signed integer type is used to represent a 32-bit float. No
+    // clamping information is provided, so the full [-128, 127] range is
+    // available. The scale is set to 3.0, and the zero point takes its default
+    // 0 value.
+    !quant.uniform<i8:f32, 3.0>
+
+    // A 16-bit unsigned integer type is used to represent a 32-bit float. Out
+    // of the 16 bits, only 10 are used, acoording to the 0..1023 clamping
+    // range. The type sets the scale to 1.23 and the zero point to 512.
+    !quant.uniform<u16<0:1023>:f32, 1.23:512>
+    ```
+
+    ### Per-channel quantization
+
+    The general syntax of the `!quant.uniform` type representing per-channel
+    quantization is as follows:
+
+    ```
+    `!quant.uniform` `<`
+      storedType (`<` storageMin `:` storageMax `>`)? `:`
+      expressedType `:`
+      channelAxis `,`
+      `{`
+        scale0 (`:` zeroPoint0)? `,`
+        scale1 (`:` zeroPoint1)? ...
+      '}'
+    `>`
+    ```
+
+    In this data type, there are multiple pairs of `scale` and `zeroPoint`
+    values. The `channelAxis` field represents the dimension of the containing
+    tensor acting as the channel. The size of the tensor along this dimension
+    is expected to match the number of provided `scale`-`zeroPoint` pairs, and
+    a given pair *i* applies to all elements in the tensor whose index along
+    dimension `channelAxis` is *i*. A quantized data type using per-channel
+    quantization is always expected to be contained within a tensor type.
+
+    Here are some examples:
+
+    ```
+    // A 2x3x4 tensor contains 8-bit signed integers representing 32-bit
+    // floats. Dimension 1 of the tensor acts as the channel dimension. Its
+    // size 3 matches the number of provided scale values. Tensor elemenets at
+    // positions [*][0][*], [*][1][*], and [*][2][*] use scales 3.0, 4.0, and
+    // 5.0, respectively.
+    tensor<2x3x4x!quant.uniform<i8:f32:1, {3.0, 4.0, 5.0}>>
+
+    // A 2D dynamically sized tensor contains 16-bit unsigned integers
+    // representing 32-bit floats. Dimension 0 of the tensor acts as the
+    // channel dimension. Since 2 scale and zero-point values are provided, the
+    // size of dimension 0 is expected to be 2 at runtime. Tensor elements
+    // [0][*] use scale 2.0 and zero point 10, while elements [1][*] use scale
+    // 3.0 and zero point 20.
+    tensor<?x?x!quant.uniform<u16:f32:0, {2.0:10, 3.0:20}>>
+    ```
+
+
+    ## Per-axis quantization integrity
+
+    When type `!quant.uniform` contains per-axis quantization information, the
+    rules below are enforced. These rules guarantee that the quantization
+    information encoded in the data type is applicable to the context in which
+    the quantized type is used. For efficiency, these rules are actively
+    enforced by the verifiers of `quant` dialect ops, but they must be
+    respected in any context in which the `!quant.uniform` data type is used,
+    such as the header of a `func.func` op, or the input of an arithmetic
+    operation.
+ 
+    - A quantized type with per-channel quantization information must be the
+      element type of a tensor container type, and may not occur directly as
+      the data type of a scalar value.
+
+    ```
+    // Incorrect. Type !quant.uniform specifies per-channel quantization for a
+    // scalar type.
+    %result = quant.qcast %input : f32 to !quant.uniform<i8:f32:0, {1.0, 2.0}>
+
+    // Correct. Type `!quant.uniform` with per-channel quantization is wrapped
+    // in a `tensor` type.
+    %result = quant.qcast %input : tensor<2xf32> to tensor<2x!quant.uniform<i8:f32:0, {1.0, 2.0}>>
+    ```
+
+    - If the tensor containing the `!quant.uniform` type is ranked, its rank
+      must be greater than the channel axis specified in the quantized type.
+
+    ```
+    // Incorrect. The tensor rank (2) is not greater than the channel axis in
+    // the quantized type (3).
+    %result = quant.qcast %input : tensor<1x2xf32> to tensor<1x2x!quant.uniform<i8:f32:3, {1.0, 2.0}>>
+
+    // Correct. The tensor rank (2) is now greater than the channel axis (1):
+    %result = quant.qcast %input : tensor<1x2xf32> to tensor<1x2x!quant.uniform<i8:f32:1, {1.0, 2.0}>>
+    ```
+
+    - If the axis dimension in the containing tensor is static, its size must
+      be equal to the number of scales present in the quantized type.
+
+    ```
+    // Incorrect. The channel axis is 1, and the size of dimension 1 in the
+    // containing tensor is 3. However, there are 4 scale values present in the
+    // quantized type.
+    %result = quant.qcast %input : tensor<?x3xf32> to tensor<?x3x!quant.uniform<i8:f32:1, {1.0, 2.0, 3.0, 4.0}>>
+
+    // Correct. The quantized type now includes 3 scale values, matching the
+    // size of dimension 1 of the result tensor.
+    %result = quant.qcast %input : tensor<?x3xf32> to tensor<?x3x!quant.uniform<i8:f32:1, {2.0, 3.0, 4.0}>>
+    ```
+  }];
+  let cppNamespace = "::mlir::quant";
+  let useDefaultTypePrinterParser = 1;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Type predicates
+//===----------------------------------------------------------------------===//
+
+class quant_ScalarOrTensorOf<Type etype> :
+    Type<Or<[etype.predicate, TensorOf<[etype]>.predicate]>,
+         "scalar or tensor of " # etype.summary>;
+
+def quant_QuantizedType :
+    Type<CPred<"::llvm::isa<mlir::quant::QuantizedType>($_self)">, "quantized type">;
+
+def quant_ScalarType :
+    Type<Or<[
+      AnySignlessInteger.predicate,
+      AnyFloat.predicate,
+      quant_QuantizedType.predicate
+    ]>,
+    "signless integer, float, or quantized scalar">;
+
+def quant_IntegerOrQuantizedType :
+    Type<Or<[
+      AnySignlessInteger.predicate,
+      quant_QuantizedType.predicate
+    ]>,
+    "signless integer or quantized type">;
+
+def quant_FloatScalarOrTensor :
+    quant_ScalarOrTensorOf<AnyFloat>;
+
+def quant_IntegerScalarOrTensor :
+    quant_ScalarOrTensorOf<AnySignlessInteger>;
+
+def quant_QuantizedScalarOrTensor :
+    quant_ScalarOrTensorOf<quant_QuantizedType>;
+
+def quant_IntegerOrQuantizedScalarOrTensor :
+    quant_ScalarOrTensorOf<quant_IntegerOrQuantizedType>;
+
+
+//===----------------------------------------------------------------------===//
+// Traits
+//===----------------------------------------------------------------------===//
+
+def quant_SameScalarOrTensorShape :
+    PredOpTrait<
+      "input and result are both scalars or both tensors with matching shape",
+      Or<[
+        And<[
+          TypeIsPred<"input", quant_ScalarType>,
+          TypeIsPred<"result", quant_ScalarType>
+        ]>,
+        And<[
+          TypeIsPred<"input", AnyUnrankedTensor>,
+          TypeIsPred<"result", AnyUnrankedTensor>
+        ]>,
+        And<[
+          TypeIsPred<"input", AnyRankedTensor>,
+          TypeIsPred<"result", AnyRankedTensor>,
+          AllShapesMatch<["input", "result"]>.predicate
+        ]>
+      ]>
+    >;
+
+def quant_IntegerAndQuantizedCombination :
+    PredOpTrait<
+      "input must be integer and result must be quantized, or vice versa",
+      Or<[
+        And<[
+          TypeIsPred<"input", quant_QuantizedScalarOrTensor>,
+          TypeIsPred<"result", quant_IntegerScalarOrTensor>
+        ]>,
+        And<[
+          TypeIsPred<"input", quant_IntegerScalarOrTensor>,
+          TypeIsPred<"result", quant_QuantizedScalarOrTensor>
+        ]>
+      ]>
+    >;
+
+#endif // QUANT_BASE
diff --git a/mlir/include/mlir/Dialect/Quant/QuantDialectBytecode.td b/mlir/include/mlir/Dialect/Quant/IR/QuantDialectBytecode.td
similarity index 100%
rename from mlir/include/mlir/Dialect/Quant/QuantDialectBytecode.td
rename to mlir/include/mlir/Dialect/Quant/IR/QuantDialectBytecode.td
diff --git a/mlir/include/mlir/Dialect/Quant/IR/QuantOps.td b/mlir/include/mlir/Dialect/Quant/IR/QuantOps.td
new file mode 100644
index 0000000000000..6ef925146dce6
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/IR/QuantOps.td
@@ -0,0 +1,243 @@
+//===- QuantOps.td - Quantization operation definition -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the operation definition file for Quantization.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef QUANT_OPS
+#define QUANT_OPS
+
+include "mlir/Dialect/Quant/IR/QuantBase.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+//===----------------------------------------------------------------------===//
+// Base classes
+//===----------------------------------------------------------------------===//
+
+class quant_Op<string mnemonic, list<Trait> traits> :
+    Op<Quant_Dialect, mnemonic, traits>;
+
+//===----------------------------------------------------------------------===//
+// Quantization casts
+//===----------------------------------------------------------------------===//
+
+def quant_DequantizeCastOp : quant_Op<"dcast", [
+    Pure,
+    quant_SameScalarOrTensorShape]> {
+  let summary = "Dequantize cast operation";
+  let description = [{
+    Convert an input quantized value into its expressed floating-point value.
+    The dequantization process consists of the following steps:
+
+    ```
+    def dequantize(quantizedValue: quantizedType) -> expressedType:
+        storedValue = reinterpretCast(quantizedValue, storageType)
+        storedValueFloat = convertIntToFloat(storedValue, expressedType)
+        zeroPointFloat = convertIntToFloat(zeroPoint, expressedType)
+        expressedValue = (storedValueFloat - zeroPointFloat) * scale
+        return expressedValue
+    ```
+
+    Here, `storageType`, `expressedType`, `scale`, and `zeroPoint` are obtained
+    from the corresponding parameters encoded in `quantizedType`. For
+    per-channel quantization, the appropriate `scale` and `zeroPoint` values
+    are used for each tensor element computation according to the channel the
+    element belongs to.
+    
+    The numerical results produced by the algorithm above may vary depending on
+    the rounding methods used by `convertIntToFloat()`, subtraction (`-`), and
+    multiplication (`*`). This operation does not define specific rounding
+    methods; instead, it is the responsibility of a transform pipeline to
+    determine which rounding method to apply when this operation is broken down
+    into lower-level dialects.
+
+    The operation must satisfy the following syntactic constraints:
+
+    - Operand `input` must be a scalar or tensor of type `!quant.uniform`.
+
+    - The result type must be a floating-point scalar or tensor.
+
+    - The `expressedType` parameter of the `!quant.uniform` type of the input
+      must match the floating-point type of the result.
+
+    - The operand and result types must be both scalars or both tensors. If
+      tensors, they must be both ranked or both unranked. If ranked, both must
+      have the same shape, including matching static and dynamic dimensions.
+
+    - If the operand uses per-channel quantization, its `!quant.uniform` type
+      must adhere to the [Per-axis quantization
+      integrity](#per-axis-quantization-integrity) guidelines.
+
+    Examples:
+
+    ```
+    // Dequantize a scalar quantized value
+    %result = quant.dcast %input : !quant.uniform<i8:f32, 2.0> to f32
+
+    // Dequantize a dynamically shaped tensor of quantized values
+    %result = quant.dcast %input : tensor<?x!quant.uniform<i8:f32, 2.0>> to tensor<?xf32>
+
+    // Dequantize an unranked tensor using per-axis quantization information
+    %result = quant.dcast %input : tensor<*x!quant.uniform<i8:f32:1, {2.0, 3.0}>> to tensor<*xf32>
+    ```
+  }];
+  let arguments = (ins quant_QuantizedScalarOrTensor:$input);
+  let results = (outs quant_FloatScalarOrTensor:$result);
+  let assemblyFormat = "$input attr-dict `:` type($input) `to` type($result)";
+  let hasVerifier = 1;
+  let hasFolder = 1;
+  let extraClassDeclaration = [{
+    /// Return the float type of the scalar or tensor result.
+    FloatType getFloatType();
+    
+    /// Return the quantized type of the scalar or tensor input.
+    quant::QuantizedType getQuantizedType();
+  }];
+}
+
+def quant_QuantizeCastOp : quant_Op<"qcast", [
+    Pure,
+    quant_SameScalarOrTensorShape]> {
+  let summary = "Quantize cast operation";
+  let description = [{
+    Convert a floating-point value to a quantized type. The quantization
+    process consists of the following steps:
+
+    ```
+    def quantize(expressedValue: expressedType) -> quantizedType:
+        zeroPointFloat = convertIntToFloat(zeroPoint, expressedType)
+        scaledValue = expressedValue / scale
+        storedValueFloat = scaledValue + zeroPointFloat
+        storedValue = convertFloatToInt(storedValueFloat, storageType)
+        storedValueClamped = clamp(storedValue, storageMin, storageMax)
+        quantizedValue = reinterpretCast(storedValueClamped, quantizedType)
+        return quantizedValue
+    ```
+
+    Here, `storageType`, `storageMin`, `storageMax`, `expressedType`, `scale`,
+    and `zeroPoint` are obtained from the corresponding parameters encoded in
+    `quantizedType`. For per-channel quantization, the appropriate `scale` and
+    `zeroPoint` values are used for each tensor element computation according
+    to the channel the element belongs to.
+
+    The numerical results produced by the algorithm above may vary depending on
+    the rounding methods used by `convertIntToFloat()`, `convertFloatToInt()`,
+    `clamp()`, division (`/`), and addition (`+`). This operation does not
+    define specific rounding methods; instead, it is the responsibility of a
+    transform pipeline to determine which rounding method to apply when this
+    operation is broken down into lower-level dialects.
+
+    The operation must satisfy the following syntactic constraints:
+
+    - Operand `input` must be a floating-point scalar or tensor.
+
+    - The result type must be a scalar or tensor of type `!quant.uniform`.
+
+    - The `expressedType` parameter in the `!quant.uniform` type of the result
+      must match the floating-point type of the input.
+
+    - The operand and result types must be both scalars or both tensors. If
+      tensors, they must be both ranked or both unranked. If ranked, both must
+      have the same shape, including matching static and dynamic dimensions.
+
+    - If the result uses per-channel quantization, its `!quant.uniform` type
+      must adhere to the [Per-axis quantization
+      integrity](#per-axis-quantization-integrity) guidelines.
+
+    Examples:
+
+    ```
+    // Quantize a scalar floating-point value
+    %result = quant.qcast %input : f32 to !quant.uniform<i8:f32, 2.0>
+
+    // Quantize a dynamically shaped tensor of quantized values
+    %result = quant.qcast %input : tensor<?xf32> to tensor<?x!quant.uniform<i8:f32, 2.0>>
+
+    // Quantize an unranked tensor using per-axis quantization information
+    %result = quant.qcast %input : tensor<*xf32> to tensor<*x!quant.uniform<i8:f32:1, {2.0, 3.0}>>
+    ```
+  }];
+  let arguments = (ins quant_FloatScalarOrTensor:$input);
+  let results = (outs quant_QuantizedScalarOrTensor:$result);
+  let assemblyFormat = "$input attr-dict `:` type($input) `to` type($result)";
+  let hasVerifier = 1;
+  let hasFolder = 1;
+  let extraClassDeclaration = [{
+    /// Return the float type of the scalar or tensor input.
+    FloatType getFloatType();
+    
+    /// Return the quantized type of the scalar or tensor result.
+    quant::QuantizedType getQuantizedType();
+  }];
+}
+
+def quant_StorageCastOp : quant_Op<"scast", [
+    Pure,
+    quant_SameScalarOrTensorShape,
+    quant_IntegerAndQuantizedCombination]> {
+  let summary = "Storage cast operation";
+  let description = [{
+    Convert a value from a quantized type to the corresponding signless integer
+    storage type, or vice versa. This conversion simply involves a
+    reinterpretation of the input bits and does not involve any data
+    manipulation.
+
+    The following syntactic restrictions must be met:
+
+    - Operand `input` must be a scalar or tensor of a signless integer or
+      `!quant.uniform` type.
+
+    - The result must be a scalar or tensor of a signless integer or
+      `!quant.uniform` type.
+
+    - If the operand is a scalar or tensor of type integer, the result must be
+      a scalar or tensor of type `!quant.uniform`, and vice versa.
+
+    - The operand and result must be both scalars or both tensors. If tensors,
+      they must be both ranked or both unranked. If ranked, both must have the
+      same shape, including matching static and dynamic dimensions.
+
+    - The width of the `storageType` parameter of the quantized type of the
+      operand or result must match the width of the signless integer type of
+      the operand or result.
+
+    - If the operand or result uses per-channel quantization, its
+      `!quant.uniform` type must adhere to the [Per-axis quantization
+      integrity](#per-axis-quantization-integrity) guidelines.
+
+    Examples:
+
+    ```
+    // Cast a scalar quantized value into its storage type
+    %result = quant.scast %input : !quant.uniform<i8:f32, 2.0> to i8
+
+    // Cast a dynamically shaped tensor of quantized values into their storage type
+    %result = quant.scast %input : tensor<?x!quant.uniform<i8:f32, 2.0>> to tensor<?xi8>
+
+    // Cast an unranked tensor of signless integers into a quantized type using
+    // per-channel quantization
+    %result = quant.scast %input : tensor<*xi8> to tensor<*x!quant.uniform<i8:f32:1, {2.0, 3.0}>>
+    ```
+  }];
+  let arguments = (ins quant_IntegerOrQuantizedScalarOrTensor:$input);
+  let results = (outs quant_IntegerOrQuantizedScalarOrTensor:$result);
+  let assemblyFormat = "$input attr-dict `:` type($input) `to` type($result)";
+  let hasVerifier = 1;
+  let hasFolder = 1;
+  let extraClassDeclaration = [{
+    /// Return the integer type used either in the input or the result.
+    IntegerType getIntegerType();
+    
+    /// Return the quantized type used either in the input or the result.
+    quant::QuantizedType getQuantizedType();
+  }];
+}
+
+#endif // QUANT_OPS
diff --git a/mlir/include/mlir/Dialect/Quant/QuantTypes.h b/mlir/include/mlir/Dialect/Quant/IR/QuantTypes.h
similarity index 98%
rename from mlir/include/mlir/Dialect/Quant/QuantTypes.h
rename to mlir/include/mlir/Dialect/Quant/IR/QuantTypes.h
index 57a2aa2983365..43440ba623b9c 100644
--- a/mlir/include/mlir/Dialect/Quant/QuantTypes.h
+++ b/mlir/include/mlir/Dialect/Quant/IR/QuantTypes.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_QUANT_QUANTTYPES_H
-#define MLIR_DIALECT_QUANT_QUANTTYPES_H
+#ifndef MLIR_DIALECT_QUANT_IR_QUANTTYPES_H
+#define MLIR_DIALECT_QUANT_IR_QUANTTYPES_H
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
@@ -114,6 +114,10 @@ class QuantizedType : public Type {
   /// The maximum value that storageType can take.
   int64_t getStorageTypeMax() const;
 
+  /// Return whether the storage type has explicit min or max boundaries
+  /// different from the minimum and maximum representable values.
+  bool hasStorageTypeBounds() const;
+
   /// Gets the integral bit width that the underlying storage type can exactly
   /// represent. For integral storage types, this will just be their width.
   unsigned getStorageTypeIntegralWidth() const;
@@ -413,4 +417,4 @@ class CalibratedQuantizedType
 } // namespace quant
 } // namespace mlir
 
-#endif // MLIR_DIALECT_QUANT_QUANTTYPES_H
+#endif // MLIR_DIALECT_QUANT_IR_QUANTTYPES_H
diff --git a/mlir/include/mlir/Dialect/Quant/QuantOps.td b/mlir/include/mlir/Dialect/Quant/QuantOps.td
deleted file mode 100644
index 7937265ce2f20..0000000000000
--- a/mlir/include/mlir/Dialect/Quant/QuantOps.td
+++ /dev/null
@@ -1,103 +0,0 @@
-//===- QuantOps.td - Quantization operation definition -----*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is the operation definition file for Quantization.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef DIALECT_QUANT_QUANT_OPS_
-#define DIALECT_QUANT_QUANT_OPS_
-
-include "mlir/Dialect/Quant/QuantOpsBase.td"
-include "mlir/Interfaces/InferTypeOpInterface.td"
-include "mlir/Interfaces/SideEffectInterfaces.td"
-
-//===----------------------------------------------------------------------===//
-// Base classes
-//===----------------------------------------------------------------------===//
-
-class quant_Op<string mnemonic, list<Trait> traits> :
-    Op<Quantization_Dialect, mnemonic, traits>;
-
-//===----------------------------------------------------------------------===//
-// Quantization casts
-//===----------------------------------------------------------------------===//
-
-def quant_QuantizeCastOp : quant_Op<"qcast", [Pure]> {
-  let summary = "convert a quantizable type to a quantized type";
-  let description = [{
-    A QuantizeCast `qcast` represents a potential type shift from a quantizable
-    type to a quantized type.
-
-    At runtime, a `qcast` will apply the transformation expressed by its
-    operand and result type. For flexibility during transformation, it is also
-    possible to have a `qcast` that performs no transformation (both its
-    operand and result type are quantizable).
-
-    A `qcast` will typically originate from either:
-      a) An expressed or implied constraint in the source dialect which signals
-         that a certain level of quantization is possible or required.
-      b) An inference made by a quantization algorithm indicating that a
-         quantized representation may be acceptable.
-
-    Especially early in transformation, it is common to have pairs of
-    `qcast` and `dcast` at points where a transition to a quantized type is
-    required. In addition, it is also common to have an identity `qcast`
-    (where the operand and result type are not quantized) at all points where
-    it is legal to use a quantized representation (but is not known to be
-    acceptable).
-  }];
-  let arguments = (ins quant_RealValueType:$arg);
-  let results = (outs quant_RealValueType:$res);
-}
-
-def quant_DequantizeCastOp : quant_Op<"dcast", [Pure]> {
-  let summary = "convert back from a quantized to quantizable (expressed) type operation";
-  let description = [{
-    A DequantizeCast op `dcast` represents the inverse of a `qcast`,
-    converting back from a quantized to quantizable (expressed) type.
-
-    Like `qcast`s, a `dcast` is allowed to have both its operand and result
-    as non quantized types. This facilitates transformations and marks edges
-    where the computation must be carried out in the expressed type.
-
-    Especially early in transformation, it is common to have `dcast`s on
-    all operands to ops that must operate with the expressed type (typically
-    math ops prior to lowering to target-specific, quantized kernels).
-  }];
-  let arguments = (ins quant_RealValueType:$arg);
-  let results = (outs quant_RealValueType:$res);
-}
-
-def quant_StorageCastOp : quant_Op<"scast", [Pure]> {
-  let summary = "cast from or to a type based on the storage type and the corresponding quantized type";
-  let description = [{
-    A StorageCast `scast` represents a cast from or to a type based on the
-    storage type and a type based on a corresponding quantized type.
-
-    This op exists to ensure type coherency for between parts of the computation
-    which are operating directly on an underlying storage type and those which
-    operate on quantized values.
-
-    Examples from storage to quantized type:
-    ```
-    i8 -> !quant<"uniform[i8:f32]{1.0}">
-    ```
-    ```
-    tensor<4xi8> -> tensor<4x!quant<"uniform[i8:f32]{1.0}">>
-    ```
-    ```
-    vector<4xi8> -> vector<4x!quant<"uniform[i8:f32]{1.0}">>
-    ```
-  }];
-  let arguments = (ins quant_RealOrStorageValueType:$arg);
-  let results = (outs quant_RealOrStorageValueType:$res);
-  let hasFolder = 1;
-}
-
-#endif // DIALECT_QUANT_QUANT_OPS_
diff --git a/mlir/include/mlir/Dialect/Quant/QuantOpsBase.td b/mlir/include/mlir/Dialect/Quant/QuantOpsBase.td
deleted file mode 100644
index da822d0a61deb..0000000000000
--- a/mlir/include/mlir/Dialect/Quant/QuantOpsBase.td
+++ /dev/null
@@ -1,74 +0,0 @@
-//===- QuantOpsBase.td - Quantization dialect base ---------*- tablegen -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Predicates for types in the Quantization dialect.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef DIALECT_QUANT_QUANT_OPS_BASE_
-#define DIALECT_QUANT_QUANT_OPS_BASE_
-
-include "mlir/IR/OpBase.td"
-
-def Quantization_Dialect : Dialect {
-  let name = "quant";
-  let cppNamespace = "::mlir::quant";
-
-  let useDefaultTypePrinterParser = 1;
-}
-
-//===----------------------------------------------------------------------===//
-// Quantization type definitions
-//===----------------------------------------------------------------------===//
-
-class quant_TypedPrimitiveOrContainer<Type etype> :
-    Type<Or<[etype.predicate,
-                TensorOf<[etype]>.predicate,
-                VectorOf<[etype]>.predicate]>,
-         "primitive/tensor/vector of " # etype.summary>;
-
-// An implementation of QuantizedType.
-def quant_QuantizedType :
-    Type<CPred<"::llvm::isa<mlir::quant::QuantizedType>($_self)">, "QuantizedType">;
-
-// A primitive type that can represent a real value. This is either a
-// floating point value or a quantized type.
-def quant_RealPrimitiveType :
-    Type<Or<[AnyFloat.predicate, quant_QuantizedType.predicate]>,
-    "real valued primitive (float or quantized type)">;
-
-// A primitive type that can represent a storage value. This is either an
-// integer or quantized type.
-def quant_StoragePrimitiveType :
-    Type<Or<[AnySignlessInteger.predicate, quant_QuantizedType.predicate]>,
-    "quantized storage primitive (integer or quantized type)">;
-
-// A primitive or container of RealPrimitiveType.
-def quant_RealValueType :
-    quant_TypedPrimitiveOrContainer<quant_RealPrimitiveType>;
-
-// A primitive or container of StoragePrimitiveType.
-def quant_StorageValueType :
-    quant_TypedPrimitiveOrContainer<quant_StoragePrimitiveType>;
-
-// Either a real valued or storage primitive or container type.
-def quant_RealOrStorageValueType :
-    Type<Or<[quant_RealValueType.predicate, quant_StorageValueType.predicate]>,
-    "real valued or storage primitive or container type">;
-
-// An implementation of UniformQuantizedType.
-def quant_UniformQuantizedType :
-    DialectType<Quantization_Dialect,
-                CPred<"::llvm::isa<UniformQuantizedType>($_self)">,
-                "UniformQuantizedType">;
-
-// Predicate for detecting a container or primitive of UniformQuantizedType.
-def quant_UniformQuantizedValueType :
-    quant_TypedPrimitiveOrContainer<quant_UniformQuantizedType>;
-
-#endif // DIALECT_QUANT_QUANT_OPS_BASE_
diff --git a/mlir/include/mlir/Dialect/Quant/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/Quant/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..30f7c1696bdb9
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/Transforms/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Quant)
+add_public_tablegen_target(MLIRQuantTransformsIncGen)
+
+add_mlir_doc(Passes QuantPasses ./ -gen-pass-doc)
diff --git a/mlir/include/mlir/Dialect/Quant/Transforms/Passes.h b/mlir/include/mlir/Dialect/Quant/Transforms/Passes.h
new file mode 100644
index 0000000000000..84be2a21b34ed
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/Transforms/Passes.h
@@ -0,0 +1,29 @@
+//===- Passes.h - Pass Entrypoints ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_QUANT_TRANSFORMS_PASSES_H_
+#define MLIR_DIALECT_QUANT_TRANSFORMS_PASSES_H_
+
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+namespace quant {
+
+#define GEN_PASS_DECL
+#include "mlir/Dialect/Quant/Transforms/Passes.h.inc"
+
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Quant/Transforms/Passes.h.inc"
+
+void populateLowerQuantOpsPatterns(RewritePatternSet &patterns);
+
+} // namespace quant
+} // namespace mlir
+
+#endif // MLIR_DIALECT_QUANT_TRANSFORMS_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/Quant/Transforms/Passes.td b/mlir/include/mlir/Dialect/Quant/Transforms/Passes.td
new file mode 100644
index 0000000000000..b25296d4db5a9
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Quant/Transforms/Passes.td
@@ -0,0 +1,49 @@
+//===-- Passes.td - Arith pass definition file --------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_QUANT_TRANSFORMS_PASSES
+#define MLIR_DIALECT_QUANT_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def LowerQuantOps : Pass<"lower-quant-ops", "func::FuncOp"> {
+  let summary = "Lower quant.dcast and quant.qcast ops";
+  let description = [{
+    Lower quantization (`quant.qcast`) and dequantization (`quant.dcast`) ops
+    into other core dialects.
+
+    The lowering process generates storage type casts in the form of
+    `quant.scast` ops to act as an interface between the original quantized
+    types of operands and results and their corresponding storage types used in
+    the generated arithmetic computations.
+  }];
+  let dependentDialects = [
+    "arith::ArithDialect",
+    "linalg::LinalgDialect",
+    "quant::QuantDialect",
+    "shape::ShapeDialect",
+    "tensor::TensorDialect"
+  ];
+}
+
+def StripFuncQuantTypes : Pass<"strip-func-quant-types"> {
+  let summary = "Strip quantized types from function headers";
+  let description = [{
+    Identify occurrences of function arguments using a quantized type and
+    replace them with a new value of the corresponding storage (signless
+    integer) type. For each converted argument, a `quant.scast` op is introduced
+    at the head of the function's entry block converting the new integer
+    argument into the original quantized value.
+  }];
+  let dependentDialects = [
+    "func::FuncDialect",
+    "quant::QuantDialect"
+  ];
+}
+
+#endif // MLIR_DIALECT_QUANT_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Quant/FakeQuantSupport.h b/mlir/include/mlir/Dialect/Quant/Utils/FakeQuantSupport.h
similarity index 93%
rename from mlir/include/mlir/Dialect/Quant/FakeQuantSupport.h
rename to mlir/include/mlir/Dialect/Quant/Utils/FakeQuantSupport.h
index 367d468b2acf1..6551efc6242a6 100644
--- a/mlir/include/mlir/Dialect/Quant/FakeQuantSupport.h
+++ b/mlir/include/mlir/Dialect/Quant/Utils/FakeQuantSupport.h
@@ -34,10 +34,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_QUANT_FAKEQUANTSUPPORT_H_
-#define MLIR_DIALECT_QUANT_FAKEQUANTSUPPORT_H_
+#ifndef MLIR_DIALECT_QUANT_UTILS_FAKEQUANTSUPPORT_H_
+#define MLIR_DIALECT_QUANT_UTILS_FAKEQUANTSUPPORT_H_
 
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 
 namespace mlir {
 namespace quant {
@@ -64,4 +64,4 @@ fakeQuantAttrsToType(Location loc, unsigned numBits, int32_t quantizedDimension,
 } // namespace quant
 } // namespace mlir
 
-#endif // MLIR_DIALECT_QUANT_FAKEQUANTSUPPORT_H_
+#endif // MLIR_DIALECT_QUANT_UTILS_FAKEQUANTSUPPORT_H_
diff --git a/mlir/include/mlir/Dialect/Quant/UniformSupport.h b/mlir/include/mlir/Dialect/Quant/Utils/UniformSupport.h
similarity index 97%
rename from mlir/include/mlir/Dialect/Quant/UniformSupport.h
rename to mlir/include/mlir/Dialect/Quant/Utils/UniformSupport.h
index 4119aced4c075..6773f45069c87 100644
--- a/mlir/include/mlir/Dialect/Quant/UniformSupport.h
+++ b/mlir/include/mlir/Dialect/Quant/Utils/UniformSupport.h
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_QUANT_UNIFORMSUPPORT_H_
-#define MLIR_DIALECT_QUANT_UNIFORMSUPPORT_H_
+#ifndef MLIR_DIALECT_QUANT_UTILS_UNIFORMSUPPORT_H_
+#define MLIR_DIALECT_QUANT_UTILS_UNIFORMSUPPORT_H_
 
 #include <utility>
 
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APFloat.h"
@@ -218,4 +218,4 @@ class UniformQuantizedPerAxisValueConverter {
 } // namespace quant
 } // namespace mlir
 
-#endif // MLIR_DIALECT_QUANT_UNIFORMSUPPORT_H_
+#endif // MLIR_DIALECT_QUANT_UTILS_UNIFORMSUPPORT_H_
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCF.h b/mlir/include/mlir/Dialect/SCF/IR/SCF.h
index 644118ca884c6..b62c941797947 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCF.h
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCF.h
@@ -107,6 +107,17 @@ LoopNest buildLoopNest(OpBuilder &builder, Location loc, ValueRange lbs,
                        function_ref<void(OpBuilder &, Location, ValueRange)>
                            bodyBuilder = nullptr);
 
+/// Perform a replacement of one iter OpOperand of an scf.for to the
+/// `replacement` value with a different type. A callback is used to insert
+/// cast ops inside the block to account for type differences.
+using ValueTypeCastFnTy =
+    llvm::function_ref<Value(OpBuilder &, Location loc, Type, Value)>;
+SmallVector<Value> replaceAndCastForOpIterArg(RewriterBase &rewriter,
+                                              scf::ForOp forOp,
+                                              OpOperand &operand,
+                                              Value replacement,
+                                              const ValueTypeCastFnTy &castFn);
+
 } // namespace scf
 } // namespace mlir
 #endif // MLIR_DIALECT_SCF_SCF_H
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
index 64bacd0e432fe..67b41187e5bfb 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOpBase.td
@@ -40,7 +40,7 @@ def Tosa_Dialect : Dialect {
     there will be tools to lower from the ML frameworks into TOSA.
   }];
 
-  let dependentDialects = ["tensor::TensorDialect", "quant::QuantizationDialect"];
+  let dependentDialects = ["tensor::TensorDialect", "quant::QuantDialect"];
 
   let cppNamespace = "mlir::tosa";
   let hasConstantMaterializer = 1;
diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
index 539b7cd0b7426..07402c8695b38 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaOps.td
@@ -881,7 +881,7 @@ def Tosa_TableOp : Tosa_InferShapedTypeOp<"table"> {
   }];
 
   let arguments = (ins
-    Tosa_Tensor: $input,
+    Tosa_Tensor: $input1,
     Tosa_Tensor1D: $table
   );
 
@@ -890,7 +890,7 @@ def Tosa_TableOp : Tosa_InferShapedTypeOp<"table"> {
   );
 
   let assemblyFormat = [{
-    $input `,` $table attr-dict `:` `(` type($input) `,` type($table) `)` `->` type($output)
+    $input1 `,` $table attr-dict `:` `(` type($input1) `,` type($table) `)` `->` type($output)
   }];
 
   let hasVerifier = 1;
@@ -1640,7 +1640,7 @@ def Tosa_ReverseOp: Tosa_Op<"reverse", [
   }];
 
   let arguments = (ins
-    Tosa_Tensor:$input,
+    Tosa_Tensor:$input1,
     I32Attr:$axis
   );
 
@@ -1667,7 +1667,7 @@ def Tosa_SliceOp : Tosa_InferShapedTypeOp<"slice"> {
   }];
 
   let arguments = (ins
-    Tosa_Tensor:$input,
+    Tosa_Tensor:$input1,
     DenseI64ArrayAttr:$start,
     DenseI64ArrayAttr:$size
   );
diff --git a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
index 298c97015fe2e..5e80745777b3b 100644
--- a/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
+++ b/mlir/include/mlir/Dialect/Tosa/Utils/QuantUtils.h
@@ -16,8 +16,8 @@
 
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 
-#include "mlir/Dialect/Quant/FakeQuantSupport.h"
-#include "mlir/Dialect/Quant/UniformSupport.h"
+#include "mlir/Dialect/Quant/Utils/FakeQuantSupport.h"
+#include "mlir/Dialect/Quant/Utils/UniformSupport.h"
 
 namespace mlir {
 namespace tosa {
diff --git a/mlir/include/mlir/IR/AffineMap.h b/mlir/include/mlir/IR/AffineMap.h
index 676da6d176497..e30950bbf292d 100644
--- a/mlir/include/mlir/IR/AffineMap.h
+++ b/mlir/include/mlir/IR/AffineMap.h
@@ -354,6 +354,24 @@ class AffineMap {
   /// returns the resulting values. `this` must be symbol-less.
   SmallVector<int64_t, 4> compose(ArrayRef<int64_t> values) const;
 
+  /// Returns the number of "zero" results (constant values == 0) in this map.
+  ///
+  /// Example:
+  ///   * For `(d0, d1) -> (d0, d1, 0)` returns 1
+  ///   * For `(d0, d1, d2) -> (d0, d1)` returns 0
+  ///   * For `(d0, d1, d2) -> (d0, 0, d1, 0, d2)` returns 2
+  size_t getNumOfZeroResults() const;
+
+  /// Returns the AffineMap resulting from removing "zero" results (constant
+  /// values == 0) from this map.
+  ///
+  /// Example:
+  ///   * For `(d0, d1) -> (d0, d1, 0)` returns `(d0, d1) -> (d0, d1)`
+  ///   * For `(d0, d1, d2) -> (d0, d1)` returns `(d0, d1, d2) -> (d0, d1)`
+  ///   * For `(d0, d1, d2) -> (d0, 0, d1, 0, d2)` returns
+  ///     `(d0, d1, d2) -> (d0, d1, d2)`
+  AffineMap dropZeroResults();
+
   /// Returns true if the AffineMap represents a subset (i.e. a projection) of a
   /// symbol-less permutation map. `allowZeroInResults` allows projected
   /// permutation maps with constant zero result expressions.
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index 73dccdb017ee1..7fd0432ddce1b 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -65,7 +65,7 @@
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/Dialect/Polynomial/IR/PolynomialDialect.h"
 #include "mlir/Dialect/Ptr/IR/PtrDialect.h"
-#include "mlir/Dialect/Quant/QuantOps.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/SCF/TransformOps/SCFTransformOps.h"
@@ -137,7 +137,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
                   pdl_interp::PDLInterpDialect,
                   polynomial::PolynomialDialect,
                   ptr::PtrDialect,
-                  quant::QuantizationDialect,
+                  quant::QuantDialect,
                   ROCDL::ROCDLDialect,
                   scf::SCFDialect,
                   shape::ShapeDialect,
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index 1b9c1b193ace6..dd8b292a87344 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -35,6 +35,7 @@
 #include "mlir/Dialect/Mesh/Transforms/Passes.h"
 #include "mlir/Dialect/NVGPU/Transforms/Passes.h"
 #include "mlir/Dialect/OpenACC/Transforms/Passes.h"
+#include "mlir/Dialect/Quant/Transforms/Passes.h"
 #include "mlir/Dialect/SCF/Transforms/Passes.h"
 #include "mlir/Dialect/SPIRV/Transforms/Passes.h"
 #include "mlir/Dialect/Shape/Transforms/Passes.h"
@@ -82,6 +83,7 @@ inline void registerAllPasses() {
   memref::registerMemRefPasses();
   mesh::registerMeshPasses();
   ml_program::registerMLProgramPasses();
+  quant::registerQuantPasses();
   registerSCFPasses();
   registerShapePasses();
   spirv::registerSPIRVPasses();
diff --git a/mlir/lib/CAPI/Dialect/Quant.cpp b/mlir/lib/CAPI/Dialect/Quant.cpp
index 0a7181d8bc17c..c94dbb5692fdb 100644
--- a/mlir/lib/CAPI/Dialect/Quant.cpp
+++ b/mlir/lib/CAPI/Dialect/Quant.cpp
@@ -8,12 +8,12 @@
 
 #include "mlir-c/Dialect/Quant.h"
 #include "mlir/CAPI/Registration.h"
-#include "mlir/Dialect/Quant/QuantOps.h"
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 
 using namespace mlir;
 
-MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(quant, quant, quant::QuantizationDialect)
+MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(quant, quant, quant::QuantDialect)
 
 //===---------------------------------------------------------------------===//
 // QuantizedType
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 4c2e8682285c5..2cc77e8fd41b9 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -544,6 +544,10 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp->getAttrs());
 
+    newOp.getProperties().operandSegmentSizes = {
+        static_cast<int32_t>(promoted.size()), 0};
+    newOp.getProperties().op_bundle_sizes = rewriter.getDenseI32ArrayAttr({});
+
     SmallVector<Value, 4> results;
     if (numResults < 2) {
       // If < 2 results, packing did not do anything and we can just return.
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 41a3ac76df4b7..739a34e0aa610 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -316,6 +316,53 @@ struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
   }
 };
 
+//===----------------------------------------------------------------------===//
+// Subgroup query ops.
+//===----------------------------------------------------------------------===//
+
+template <typename SubgroupOp>
+struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
+  using ConvertOpToLLVMPattern<SubgroupOp>::ConvertOpToLLVMPattern;
+  using ConvertToLLVMPattern::getTypeConverter;
+
+  LogicalResult
+  matchAndRewrite(SubgroupOp op, typename SubgroupOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const final {
+    constexpr StringRef funcName = [] {
+      if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupIdOp>) {
+        return "_Z16get_sub_group_id";
+      } else if constexpr (std::is_same_v<SubgroupOp, gpu::LaneIdOp>) {
+        return "_Z22get_sub_group_local_id";
+      } else if constexpr (std::is_same_v<SubgroupOp, gpu::NumSubgroupsOp>) {
+        return "_Z18get_num_sub_groups";
+      } else if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupSizeOp>) {
+        return "_Z18get_sub_group_size";
+      }
+    }();
+
+    Operation *moduleOp =
+        op->template getParentWithTrait<OpTrait::SymbolTable>();
+    Type resultTy = rewriter.getI32Type();
+    LLVM::LLVMFuncOp func =
+        lookupOrCreateSPIRVFn(moduleOp, funcName, {}, resultTy,
+                              /*isMemNone=*/false, /*isConvergent=*/false);
+
+    Location loc = op->getLoc();
+    Value result = createSPIRVBuiltinCall(loc, rewriter, func, {}).getResult();
+
+    Type indexTy = getTypeConverter()->getIndexType();
+    if (resultTy != indexTy) {
+      if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
+        return failure();
+      }
+      result = rewriter.create<LLVM::ZExtOp>(loc, indexTy, result);
+    }
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // GPU To LLVM-SPV Pass.
 //===----------------------------------------------------------------------===//
@@ -337,7 +384,9 @@ struct GPUToLLVMSPVConversionPass final
 
     target.addIllegalOp<gpu::BarrierOp, gpu::BlockDimOp, gpu::BlockIdOp,
                         gpu::GPUFuncOp, gpu::GlobalIdOp, gpu::GridDimOp,
-                        gpu::ReturnOp, gpu::ShuffleOp, gpu::ThreadIdOp>();
+                        gpu::LaneIdOp, gpu::NumSubgroupsOp, gpu::ReturnOp,
+                        gpu::ShuffleOp, gpu::SubgroupIdOp, gpu::SubgroupSizeOp,
+                        gpu::ThreadIdOp>();
 
     populateGpuToLLVMSPVConversionPatterns(converter, patterns);
     populateGpuMemorySpaceAttributeConversions(converter);
@@ -366,11 +415,15 @@ gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) {
 void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
                                             RewritePatternSet &patterns) {
   patterns.add<GPUBarrierConversion, GPUReturnOpLowering, GPUShuffleConversion,
+               GPUSubgroupOpConversion<gpu::LaneIdOp>,
+               GPUSubgroupOpConversion<gpu::NumSubgroupsOp>,
+               GPUSubgroupOpConversion<gpu::SubgroupIdOp>,
+               GPUSubgroupOpConversion<gpu::SubgroupSizeOp>,
+               LaunchConfigOpConversion<gpu::BlockDimOp>,
                LaunchConfigOpConversion<gpu::BlockIdOp>,
+               LaunchConfigOpConversion<gpu::GlobalIdOp>,
                LaunchConfigOpConversion<gpu::GridDimOp>,
-               LaunchConfigOpConversion<gpu::BlockDimOp>,
-               LaunchConfigOpConversion<gpu::ThreadIdOp>,
-               LaunchConfigOpConversion<gpu::GlobalIdOp>>(typeConverter);
+               LaunchConfigOpConversion<gpu::ThreadIdOp>>(typeConverter);
   MLIRContext *context = &typeConverter.getContext();
   unsigned privateAddressSpace =
       gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Private);
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
index ca78631632419..6ae607f75adbd 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/SPIRVToLLVM.cpp
@@ -837,8 +837,11 @@ class FunctionCallPattern
   matchAndRewrite(spirv::FunctionCallOp callOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     if (callOp.getNumResults() == 0) {
-      rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+      auto newOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
           callOp, std::nullopt, adaptor.getOperands(), callOp->getAttrs());
+      newOp.getProperties().operandSegmentSizes = {
+          static_cast<int32_t>(adaptor.getOperands().size()), 0};
+      newOp.getProperties().op_bundle_sizes = rewriter.getDenseI32ArrayAttr({});
       return success();
     }
 
@@ -846,8 +849,11 @@ class FunctionCallPattern
     auto dstType = typeConverter.convertType(callOp.getType(0));
     if (!dstType)
       return rewriter.notifyMatchFailure(callOp, "type conversion failed");
-    rewriter.replaceOpWithNewOp<LLVM::CallOp>(
+    auto newOp = rewriter.replaceOpWithNewOp<LLVM::CallOp>(
         callOp, dstType, adaptor.getOperands(), callOp->getAttrs());
+    newOp.getProperties().operandSegmentSizes = {
+        static_cast<int32_t>(adaptor.getOperands().size()), 0};
+    newOp.getProperties().op_bundle_sizes = rewriter.getDenseI32ArrayAttr({});
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 93e284af05188..01fdd57260797 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -1830,7 +1830,7 @@ class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
   LogicalResult matchAndRewrite(tosa::ReverseOp op,
                                 PatternRewriter &rewriter) const final {
     auto loc = op.getLoc();
-    Value input = op.getInput();
+    Value input = op.getInput1();
     auto inputTy = cast<ShapedType>(input.getType());
     auto resultTy = cast<ShapedType>(op.getType());
     auto axis = op.getAxis();
@@ -2161,7 +2161,7 @@ class TableConverter : public OpRewritePattern<tosa::TableOp> {
   LogicalResult matchAndRewrite(tosa::TableOp op,
                                 PatternRewriter &rewriter) const final {
     auto loc = op.getLoc();
-    Value input = op.getInput();
+    Value input = op.getInput1();
     Value table = op.getTable();
     auto inputTy = cast<ShapedType>(input.getType());
     auto tableTy = cast<ShapedType>(table.getType());
diff --git a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
index c0c015ab34aab..3f104ed1e3f7f 100644
--- a/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
+++ b/mlir/lib/Conversion/TosaToTensor/TosaToTensor.cpp
@@ -144,7 +144,7 @@ TensorType inferReshapeCollapsedType(TensorType lhsType, TensorType rhsType) {
   for (; currRhsDim < rhsShape.size(); currRhsDim++) {
     assert(rhsShape[currRhsDim] == 1);
   }
-  
+
   return lhsType.clone(intermediateShape);
 }
 
@@ -264,7 +264,7 @@ class SliceConverter : public OpConversionPattern<tosa::SliceOp> {
   matchAndRewrite(tosa::SliceOp sliceOp, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const final {
     Location loc = sliceOp.getLoc();
-    Value input = adaptor.getInput();
+    Value input = adaptor.getInput1();
     ShapedType resultType = cast<ShapedType>(sliceOp.getType());
     if (llvm::isa<UnrankedTensorType>(resultType))
       return failure();
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index fa03442765539..0e21e96cc3fbb 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -106,7 +106,7 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
       std::optional<int64_t> staticVal = getConstantIntValue(offset);
       if (!staticVal)
         dynOffsets.push_back(offset);
-      constOffsets.push_back(staticVal ? *staticVal : ShapedType::kDynamic);
+      constOffsets.push_back(staticVal.value_or(ShapedType::kDynamic));
     }
 
     SmallVector<Value> dynShapes;
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index d68a29f07f1b6..150b9824c41e3 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -2067,7 +2067,7 @@ static LogicalResult generateCopy(
     // fastMemRefType is a constant shaped memref.
     auto maySizeInBytes = getIntOrFloatMemRefSizeInBytes(fastMemRefType);
     // We don't account for things of unknown size.
-    *sizeInBytes = maySizeInBytes ? *maySizeInBytes : 0;
+    *sizeInBytes = maySizeInBytes.value_or(0);
 
     LLVM_DEBUG(emitRemarkForBlock(*block)
                << "Creating fast buffer of type " << fastMemRefType
diff --git a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
index 0bf8c8942885e..b51444e884aae 100644
--- a/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/EmulateUnsupportedFloats.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Dialect/Arith/Transforms/Passes.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Location.h"
@@ -49,30 +50,6 @@ struct EmulateFloatPattern final : ConversionPattern {
 };
 } // end namespace
 
-/// Map strings to float types. This function is here because no one else needs
-/// it yet, feel free to abstract it out.
-static std::optional<FloatType> parseFloatType(MLIRContext *ctx,
-                                               StringRef name) {
-  Builder b(ctx);
-  return llvm::StringSwitch<std::optional<FloatType>>(name)
-      .Case("f4E2M1FN", b.getFloat4E2M1FNType())
-      .Case("f6E2M3FN", b.getFloat6E2M3FNType())
-      .Case("f6E3M2FN", b.getFloat6E3M2FNType())
-      .Case("f8E5M2", b.getFloat8E5M2Type())
-      .Case("f8E4M3", b.getFloat8E4M3Type())
-      .Case("f8E4M3FN", b.getFloat8E4M3FNType())
-      .Case("f8E5M2FNUZ", b.getFloat8E5M2FNUZType())
-      .Case("f8E4M3FNUZ", b.getFloat8E4M3FNUZType())
-      .Case("f8E3M4", b.getFloat8E3M4Type())
-      .Case("bf16", b.getBF16Type())
-      .Case("f16", b.getF16Type())
-      .Case("f32", b.getF32Type())
-      .Case("f64", b.getF64Type())
-      .Case("f80", b.getF80Type())
-      .Case("f128", b.getF128Type())
-      .Default(std::nullopt);
-}
-
 LogicalResult EmulateFloatPattern::match(Operation *op) const {
   if (getTypeConverter()->isLegal(op))
     return failure();
@@ -156,7 +133,8 @@ void EmulateUnsupportedFloatsPass::runOnOperation() {
   SmallVector<Type> sourceTypes;
   Type targetType;
 
-  std::optional<FloatType> maybeTargetType = parseFloatType(ctx, targetTypeStr);
+  std::optional<FloatType> maybeTargetType =
+      arith::parseFloatType(ctx, targetTypeStr);
   if (!maybeTargetType) {
     emitError(UnknownLoc::get(ctx), "could not map target type '" +
                                         targetTypeStr +
@@ -166,7 +144,7 @@ void EmulateUnsupportedFloatsPass::runOnOperation() {
   targetType = *maybeTargetType;
   for (StringRef sourceTypeStr : sourceTypeStrs) {
     std::optional<FloatType> maybeSourceType =
-        parseFloatType(ctx, sourceTypeStr);
+        arith::parseFloatType(ctx, sourceTypeStr);
     if (!maybeSourceType) {
       emitError(UnknownLoc::get(ctx), "could not map source type '" +
                                           sourceTypeStr +
diff --git a/mlir/lib/Dialect/Arith/Utils/Utils.cpp b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
index e75db84b75e28..c0aa16cc0da40 100644
--- a/mlir/lib/Dialect/Arith/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Arith/Utils/Utils.cpp
@@ -357,4 +357,26 @@ Value createProduct(OpBuilder &builder, Location loc, ArrayRef<Value> values,
       [&arithBuilder](Value acc, Value v) { return arithBuilder.mul(acc, v); });
 }
 
+/// Map strings to float types.
+std::optional<FloatType> parseFloatType(MLIRContext *ctx, StringRef name) {
+  Builder b(ctx);
+  return llvm::StringSwitch<std::optional<FloatType>>(name)
+      .Case("f4E2M1FN", b.getFloat4E2M1FNType())
+      .Case("f6E2M3FN", b.getFloat6E2M3FNType())
+      .Case("f6E3M2FN", b.getFloat6E3M2FNType())
+      .Case("f8E5M2", b.getFloat8E5M2Type())
+      .Case("f8E4M3", b.getFloat8E4M3Type())
+      .Case("f8E4M3FN", b.getFloat8E4M3FNType())
+      .Case("f8E5M2FNUZ", b.getFloat8E5M2FNUZType())
+      .Case("f8E4M3FNUZ", b.getFloat8E4M3FNUZType())
+      .Case("f8E3M4", b.getFloat8E3M4Type())
+      .Case("bf16", b.getBF16Type())
+      .Case("f16", b.getF16Type())
+      .Case("f32", b.getF32Type())
+      .Case("f64", b.getF64Type())
+      .Case("f80", b.getF80Type())
+      .Case("f128", b.getF128Type())
+      .Default(std::nullopt);
+}
+
 } // namespace mlir::arith
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 205d7494d4378..0561c364c7d59 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -220,6 +220,91 @@ static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
   return static_cast<RetTy>(index);
 }
 
+//===----------------------------------------------------------------------===//
+// Operand bundle helpers.
+//===----------------------------------------------------------------------===//
+
+static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands,
+                             TypeRange operandTypes, StringRef tag) {
+  p.printString(tag);
+  p << "(";
+
+  if (!operands.empty()) {
+    p.printOperands(operands);
+    p << " : ";
+    llvm::interleaveComma(operandTypes, p);
+  }
+
+  p << ")";
+}
+
+static void printOpBundles(OpAsmPrinter &p, Operation *op,
+                           OperandRangeRange opBundleOperands,
+                           TypeRangeRange opBundleOperandTypes,
+                           ArrayRef<std::string> opBundleTags) {
+  p << "[";
+  llvm::interleaveComma(
+      llvm::zip(opBundleOperands, opBundleOperandTypes, opBundleTags), p,
+      [&p](auto bundle) {
+        printOneOpBundle(p, std::get<0>(bundle), std::get<1>(bundle),
+                         std::get<2>(bundle));
+      });
+  p << "]";
+}
+
+static ParseResult parseOneOpBundle(
+    OpAsmParser &p,
+    SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
+    SmallVector<SmallVector<Type>> &opBundleOperandTypes,
+    SmallVector<std::string> &opBundleTags) {
+  SMLoc currentParserLoc = p.getCurrentLocation();
+  SmallVector<OpAsmParser::UnresolvedOperand> operands;
+  SmallVector<Type> types;
+  std::string tag;
+
+  if (p.parseString(&tag))
+    return p.emitError(currentParserLoc, "expect operand bundle tag");
+
+  if (p.parseLParen())
+    return failure();
+
+  if (p.parseOptionalRParen()) {
+    if (p.parseOperandList(operands) || p.parseColon() ||
+        p.parseTypeList(types) || p.parseRParen())
+      return failure();
+  }
+
+  opBundleOperands.push_back(std::move(operands));
+  opBundleOperandTypes.push_back(std::move(types));
+  opBundleTags.push_back(std::move(tag));
+
+  return success();
+}
+
+static std::optional<ParseResult> parseOpBundles(
+    OpAsmParser &p,
+    SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
+    SmallVector<SmallVector<Type>> &opBundleOperandTypes,
+    SmallVector<std::string> &opBundleTags) {
+  if (p.parseOptionalLSquare())
+    return std::nullopt;
+
+  if (succeeded(p.parseOptionalRSquare()))
+    return success();
+
+  auto bundleParser = [&] {
+    return parseOneOpBundle(p, opBundleOperands, opBundleOperandTypes,
+                            opBundleTags);
+  };
+  if (p.parseCommaSeparatedList(bundleParser))
+    return failure();
+
+  if (p.parseRSquare())
+    return failure();
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Printing, parsing, folding and builder for LLVM::CmpOp.
 //===----------------------------------------------------------------------===//
@@ -954,6 +1039,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
         /*CConv=*/nullptr, /*TailCallKind=*/nullptr,
         /*memory_effects=*/nullptr,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+        /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -980,6 +1066,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
         /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
         /*convergent=*/nullptr,
         /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+        /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
         /*access_groups=*/nullptr,
         /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -992,6 +1079,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state,
         /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
         /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+        /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -1004,6 +1092,7 @@ void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
         /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
         /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
         /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
+        /*op_bundle_operands=*/{}, /*op_bundle_tags=*/std::nullopt,
         /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
         /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
@@ -1027,7 +1116,7 @@ void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
 }
 
 Operation::operand_range CallOp::getArgOperands() {
-  return getOperands().drop_front(getCallee().has_value() ? 0 : 1);
+  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
 }
 
 MutableOperandRange CallOp::getArgOperandsMutable() {
@@ -1100,6 +1189,21 @@ LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
   return success();
 }
 
+template <typename OpType>
+static LogicalResult verifyOperandBundles(OpType &op) {
+  OperandRangeRange opBundleOperands = op.getOpBundleOperands();
+  ArrayRef<std::string> opBundleTags = op.getOpBundleTags();
+
+  if (opBundleTags.size() != opBundleOperands.size())
+    return op.emitError("expected ")
+           << opBundleOperands.size()
+           << " operand bundle tags, but actually got " << opBundleTags.size();
+
+  return success();
+}
+
+LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
+
 LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   if (failed(verifyCallOpVarCalleeType(*this)))
     return failure();
@@ -1150,15 +1254,15 @@ LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   // Verify that the operand and result types match the callee.
 
   if (!funcType.isVarArg() &&
-      funcType.getNumParams() != (getNumOperands() - isIndirect))
+      funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
     return emitOpError() << "incorrect number of operands ("
-                         << (getNumOperands() - isIndirect)
+                         << (getCalleeOperands().size() - isIndirect)
                          << ") for callee (expecting: "
                          << funcType.getNumParams() << ")";
 
-  if (funcType.getNumParams() > (getNumOperands() - isIndirect))
+  if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
     return emitOpError() << "incorrect number of operands ("
-                         << (getNumOperands() - isIndirect)
+                         << (getCalleeOperands().size() - isIndirect)
                          << ") for varargs callee (expecting at least: "
                          << funcType.getNumParams() << ")";
 
@@ -1208,16 +1312,24 @@ void CallOp::print(OpAsmPrinter &p) {
   else
     p << getOperand(0);
 
-  auto args = getOperands().drop_front(isDirect ? 0 : 1);
+  auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
   p << '(' << args << ')';
 
   // Print the variadic callee type if the call is variadic.
   if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
     p << " vararg(" << *varCalleeType << ")";
 
+  if (!getOpBundleOperands().empty()) {
+    p << " ";
+    printOpBundles(p, *this, getOpBundleOperands(),
+                   getOpBundleOperands().getTypes(), getOpBundleTags());
+  }
+
   p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
                           {getCalleeAttrName(), getTailCallKindAttrName(),
-                           getVarCalleeTypeAttrName(), getCConvAttrName()});
+                           getVarCalleeTypeAttrName(), getCConvAttrName(),
+                           getOperandSegmentSizesAttrName(),
+                           getOpBundleSizesAttrName()});
 
   p << " : ";
   if (!isDirect)
@@ -1285,14 +1397,47 @@ static ParseResult parseOptionalCallFuncPtr(
   return success();
 }
 
+static ParseResult resolveOpBundleOperands(
+    OpAsmParser &parser, SMLoc loc, OperationState &state,
+    ArrayRef<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands,
+    ArrayRef<SmallVector<Type>> opBundleOperandTypes,
+    StringAttr opBundleSizesAttrName) {
+  unsigned opBundleIndex = 0;
+  for (const auto &[operands, types] :
+       llvm::zip_equal(opBundleOperands, opBundleOperandTypes)) {
+    if (operands.size() != types.size())
+      return parser.emitError(loc, "expected ")
+             << operands.size()
+             << " types for operand bundle operands for operand bundle #"
+             << opBundleIndex << ", but actually got " << types.size();
+    if (parser.resolveOperands(operands, types, loc, state.operands))
+      return failure();
+  }
+
+  SmallVector<int32_t> opBundleSizes;
+  opBundleSizes.reserve(opBundleOperands.size());
+  for (const auto &operands : opBundleOperands)
+    opBundleSizes.push_back(operands.size());
+
+  state.addAttribute(
+      opBundleSizesAttrName,
+      DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
+
+  return success();
+}
+
 // <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
 //                             `(` ssa-use-list `)`
 //                             ( `vararg(` var-callee-type `)` )?
+//                             ( `[` op-bundles-list `]` )?
 //                             attribute-dict? `:` (type `,`)? function-type
 ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
   SymbolRefAttr funcAttr;
   TypeAttr varCalleeType;
   SmallVector<OpAsmParser::UnresolvedOperand> operands;
+  SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
+  SmallVector<SmallVector<Type>> opBundleOperandTypes;
+  SmallVector<std::string> opBundleTags;
 
   // Default to C Calling Convention if no keyword is provided.
   result.addAttribute(
@@ -1333,11 +1478,35 @@ ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
       return failure();
   }
 
+  SMLoc opBundlesLoc = parser.getCurrentLocation();
+  if (std::optional<ParseResult> result = parseOpBundles(
+          parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
+      result && failed(*result))
+    return failure();
+  if (!opBundleTags.empty())
+    result.getOrAddProperties<CallOp::Properties>().op_bundle_tags =
+        std::move(opBundleTags);
+
   if (parser.parseOptionalAttrDict(result.attributes))
     return failure();
 
   // Parse the trailing type list and resolve the operands.
-  return parseCallTypeAndResolveOperands(parser, result, isDirect, operands);
+  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
+    return failure();
+  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
+                              opBundleOperandTypes,
+                              getOpBundleSizesAttrName(result.name)))
+    return failure();
+
+  int32_t numOpBundleOperands = 0;
+  for (const auto &operands : opBundleOperands)
+    numOpBundleOperands += operands.size();
+
+  result.addAttribute(
+      CallOp::getOperandSegmentSizeAttr(),
+      parser.getBuilder().getDenseI32ArrayAttr(
+          {static_cast<int32_t>(operands.size()), numOpBundleOperands}));
+  return success();
 }
 
 LLVMFunctionType CallOp::getCalleeFunctionType() {
@@ -1356,7 +1525,8 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
   auto calleeType = func.getFunctionType();
   build(builder, state, getCallOpResultTypes(calleeType),
         getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
-        normalOps, unwindOps, nullptr, nullptr, normal, unwind);
+        normalOps, unwindOps, nullptr, nullptr, {}, std::nullopt, normal,
+        unwind);
 }
 
 void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
@@ -1365,7 +1535,7 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
                      ValueRange unwindOps) {
   build(builder, state, tys,
         /*var_callee_type=*/nullptr, callee, ops, normalOps, unwindOps, nullptr,
-        nullptr, normal, unwind);
+        nullptr, {}, std::nullopt, normal, unwind);
 }
 
 void InvokeOp::build(OpBuilder &builder, OperationState &state,
@@ -1374,7 +1544,7 @@ void InvokeOp::build(OpBuilder &builder, OperationState &state,
                      Block *unwind, ValueRange unwindOps) {
   build(builder, state, getCallOpResultTypes(calleeType),
         getCallOpVarCalleeType(calleeType), callee, ops, normalOps, unwindOps,
-        nullptr, nullptr, normal, unwind);
+        nullptr, nullptr, {}, std::nullopt, normal, unwind);
 }
 
 SuccessorOperands InvokeOp::getSuccessorOperands(unsigned index) {
@@ -1402,7 +1572,7 @@ void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
 }
 
 Operation::operand_range InvokeOp::getArgOperands() {
-  return getOperands().drop_front(getCallee().has_value() ? 0 : 1);
+  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
 }
 
 MutableOperandRange InvokeOp::getArgOperandsMutable() {
@@ -1423,6 +1593,9 @@ LogicalResult InvokeOp::verify() {
     return emitError("first operation in unwind destination should be a "
                      "llvm.landingpad operation");
 
+  if (failed(verifyOperandBundles(*this)))
+    return failure();
+
   return success();
 }
 
@@ -1442,7 +1615,7 @@ void InvokeOp::print(OpAsmPrinter &p) {
   else
     p << getOperand(0);
 
-  p << '(' << getOperands().drop_front(isDirect ? 0 : 1) << ')';
+  p << '(' << getCalleeOperands().drop_front(isDirect ? 0 : 1) << ')';
   p << " to ";
   p.printSuccessorAndUseList(getNormalDest(), getNormalDestOperands());
   p << " unwind ";
@@ -1452,15 +1625,23 @@ void InvokeOp::print(OpAsmPrinter &p) {
   if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
     p << " vararg(" << *varCalleeType << ")";
 
+  if (!getOpBundleOperands().empty()) {
+    p << " ";
+    printOpBundles(p, *this, getOpBundleOperands(),
+                   getOpBundleOperands().getTypes(), getOpBundleTags());
+  }
+
   p.printOptionalAttrDict((*this)->getAttrs(),
                           {getCalleeAttrName(), getOperandSegmentSizeAttr(),
-                           getCConvAttrName(), getVarCalleeTypeAttrName()});
+                           getCConvAttrName(), getVarCalleeTypeAttrName(),
+                           getOpBundleSizesAttrName()});
 
   p << " : ";
   if (!isDirect)
     p << getOperand(0).getType() << ", ";
-  p.printFunctionalType(llvm::drop_begin(getOperandTypes(), isDirect ? 0 : 1),
-                        getResultTypes());
+  p.printFunctionalType(
+      llvm::drop_begin(getCalleeOperands().getTypes(), isDirect ? 0 : 1),
+      getResultTypes());
 }
 
 // <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
@@ -1468,11 +1649,15 @@ void InvokeOp::print(OpAsmPrinter &p) {
 //                  `to` bb-id (`[` ssa-use-and-type-list `]`)?
 //                  `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
 //                  ( `vararg(` var-callee-type `)` )?
+//                  ( `[` op-bundles-list `]` )?
 //                  attribute-dict? `:` (type `,`)? function-type
 ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
   SmallVector<OpAsmParser::UnresolvedOperand, 8> operands;
   SymbolRefAttr funcAttr;
   TypeAttr varCalleeType;
+  SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
+  SmallVector<SmallVector<Type>> opBundleOperandTypes;
+  SmallVector<std::string> opBundleTags;
   Block *normalDest, *unwindDest;
   SmallVector<Value, 4> normalOperands, unwindOperands;
   Builder &builder = parser.getBuilder();
@@ -1513,22 +1698,40 @@ ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
       return failure();
   }
 
+  SMLoc opBundlesLoc = parser.getCurrentLocation();
+  if (std::optional<ParseResult> result = parseOpBundles(
+          parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
+      result && failed(*result))
+    return failure();
+  if (!opBundleTags.empty())
+    result.getOrAddProperties<InvokeOp::Properties>().op_bundle_tags =
+        std::move(opBundleTags);
+
   if (parser.parseOptionalAttrDict(result.attributes))
     return failure();
 
   // Parse the trailing type list and resolve the function operands.
   if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
     return failure();
+  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
+                              opBundleOperandTypes,
+                              getOpBundleSizesAttrName(result.name)))
+    return failure();
 
   result.addSuccessors({normalDest, unwindDest});
   result.addOperands(normalOperands);
   result.addOperands(unwindOperands);
 
-  result.addAttribute(InvokeOp::getOperandSegmentSizeAttr(),
-                      builder.getDenseI32ArrayAttr(
-                          {static_cast<int32_t>(operands.size()),
-                           static_cast<int32_t>(normalOperands.size()),
-                           static_cast<int32_t>(unwindOperands.size())}));
+  int32_t numOpBundleOperands = 0;
+  for (const auto &operands : opBundleOperands)
+    numOpBundleOperands += operands.size();
+
+  result.addAttribute(
+      InvokeOp::getOperandSegmentSizeAttr(),
+      builder.getDenseI32ArrayAttr({static_cast<int32_t>(operands.size()),
+                                    static_cast<int32_t>(normalOperands.size()),
+                                    static_cast<int32_t>(unwindOperands.size()),
+                                    numOpBundleOperands}));
   return success();
 }
 
@@ -3108,6 +3311,8 @@ OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
 LogicalResult CallIntrinsicOp::verify() {
   if (!getIntrin().starts_with("llvm."))
     return emitOpError() << "intrinsic name must start with 'llvm.'";
+  if (failed(verifyOperandBundles(*this)))
+    return failure();
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
index 0c48a5aeb26a2..0b5191664a9e2 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgInterfaces.cpp
@@ -222,7 +222,7 @@ bool mlir::linalg::detail::isContractionBody(
   Value contributed = getSourceSkipUnary(
       isa<BlockArgument>(reductionLHS) ? reductionRHS : reductionLHS);
   Operation *elementwiseOp = contributed.getDefiningOp();
-  if (elementwiseOp->getNumResults() != 1 ||
+  if (!elementwiseOp || elementwiseOp->getNumResults() != 1 ||
       elementwiseOp->getNumOperands() != 2) {
     errs << "expected elementwise op to be binary";
     return false;
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 29b5631f61b48..46c8510f4ed51 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -2000,7 +2000,7 @@ transform::HoistPadOp::applyToOne(transform::TransformRewriter &rewriter,
                                   transform::ApplyToEachResultList &results,
                                   transform::TransformState &state) {
   tensor::PadOp hoistedPadOp;
-  SmallVector<GenericOp> transposeOps;
+  SmallVector<TransposeOp> transposeOps;
   FailureOr<Value> result =
       hoistPaddingOnTensors(rewriter, target, getNumLoops(), getTranspose(),
                             hoistedPadOp, transposeOps);
@@ -3504,9 +3504,7 @@ DiagnosedSilenceableFailure transform::VectorizeOp::apply(
 
     if (failed(linalg::vectorize(rewriter, target, vectorSizes,
                                  getScalableSizes(),
-                                 getVectorizeNdExtract().has_value()
-                                     ? getVectorizeNdExtract().value()
-                                     : false))) {
+                                 getVectorizeNdExtract().value_or(false)))) {
       return mlir::emitSilenceableFailure(target->getLoc())
              << "Attempted to vectorize, but failed";
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index c3a08ce86082a..d33a17af63459 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -633,15 +633,15 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
                                       rewriter.getIndexAttr(1));
 
   // Step 3. Optionally transpose the padded tensor.
-  GenericOp maybeTransposeOp;
+  TransposeOp maybeTransposeOp;
   Value paddedTensor = bvm.lookup(opToHoist.getResult());
   if (!transposeVector.empty()) {
     Value outputTensor = rewriter.create<tensor::ExtractSliceOp>(
         loc, transposedTensorType, hoistedPackedTensor, offsets, sizes,
         strides);
-    maybeTransposeOp = makeTransposeOp(rewriter, loc, paddedTensor,
-                                       outputTensor, transposeVector);
-    paddedTensor = maybeTransposeOp.getResult(0);
+    maybeTransposeOp = rewriter.create<linalg::TransposeOp>(
+        loc, paddedTensor, outputTensor, transposeVector);
+    paddedTensor = maybeTransposeOp.getResult()[0];
   }
 
   // Innermost tensor.insert_slice and yields are optional / need loops.
@@ -938,7 +938,7 @@ static Value replaceByPackingResult(RewriterBase &rewriter,
 FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     RewriterBase &rewriter, tensor::PadOp opToHoist, int64_t numLoops,
     ArrayRef<int64_t> transposeVector, tensor::PadOp &hoistedOp,
-    SmallVectorImpl<GenericOp> &transposeOps) {
+    SmallVectorImpl<TransposeOp> &transposeOps) {
   LLVM_DEBUG(DBGS() << "\n"; DBGS() << " Try to hoist " << *(opToHoist) << "\n";
              DBGS() << " by " << numLoops << " loops\n");
 
@@ -980,9 +980,9 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
     // Transpose the packed tensor back to the original storage order.
     Value emptyTensor = rewriter.create<tensor::EmptyOp>(
         loc, paddedTensorType.getShape(), paddedTensorType.getElementType());
-    GenericOp unTransposeOp =
-        makeTransposeOp(rewriter, loc, newResult, emptyTensor, transposeVector);
-    newResult = unTransposeOp.getResult(0);
+    TransposeOp unTransposeOp = rewriter.create<linalg::TransposeOp>(
+        loc, newResult, emptyTensor, transposeVector);
+    newResult = unTransposeOp.getResult()[0];
     transposeOps.push_back(unTransposeOp);
   }
 
@@ -999,11 +999,10 @@ FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
   return newResult;
 }
 
-FailureOr<Value>
-mlir::linalg::hoistPaddingOnTensors(tensor::PadOp opToHoist, int64_t numLoops,
-                                    ArrayRef<int64_t> transposeVector,
-                                    tensor::PadOp &hoistedOp,
-                                    SmallVectorImpl<GenericOp> &transposeOps) {
+FailureOr<Value> mlir::linalg::hoistPaddingOnTensors(
+    tensor::PadOp opToHoist, int64_t numLoops,
+    ArrayRef<int64_t> transposeVector, tensor::PadOp &hoistedOp,
+    SmallVectorImpl<TransposeOp> &transposeOps) {
   IRRewriter rewriter(opToHoist.getContext());
   return hoistPaddingOnTensors(rewriter, opToHoist, numLoops, transposeVector,
                                hoistedOp, transposeOps);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index 518d2e138c02a..a066c44408915 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -299,7 +299,7 @@ mlir::linalg::padAndHoistLinalgOp(RewriterBase &rewriter, LinalgOp linalgOp,
     }
 
     tensor::PadOp hoistedOp;
-    SmallVector<GenericOp> transposeOps;
+    SmallVector<TransposeOp> transposeOps;
     SmallVector<int64_t> transposeVector =
         en.index() < options.transposePaddings.size()
             ? options.transposePaddings[en.index()]
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index fa20001f66182..ca85f4b9b9c15 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -224,10 +224,10 @@ struct VectorizationState {
   /// Masks an operation with the canonical vector mask if the operation needs
   /// masking. Returns the masked operation or the original operation if masking
   /// is not needed. If provided, the canonical mask for this operation is
-  /// permuted using `maybeMaskingMap`.
+  /// permuted using `maybeIndexingMap`.
   Operation *
   maskOperation(RewriterBase &rewriter, Operation *opToMask, LinalgOp linalgOp,
-                std::optional<AffineMap> maybeMaskingMap = std::nullopt);
+                std::optional<AffineMap> maybeIndexingMap = std::nullopt);
 
 private:
   /// Initializes the iteration space static sizes using the Linalg op
@@ -422,16 +422,28 @@ Value VectorizationState::getOrCreateMaskFor(
   return mask;
 }
 
-/// Masks an operation with the canonical vector mask if the operation needs
-/// masking. Returns the masked operation or the original operation if masking
-/// is not needed. If provided, the canonical mask for this operation is
-/// permuted using `maybeMaskingMap`.
 Operation *
 VectorizationState::maskOperation(RewriterBase &rewriter, Operation *opToMask,
                                   LinalgOp linalgOp,
-                                  std::optional<AffineMap> maybeMaskingMap) {
+                                  std::optional<AffineMap> maybeIndexingMap) {
   LDBG("Trying to mask: " << *opToMask << "\n");
 
+  std::optional<AffineMap> maybeMaskingMap = std::nullopt;
+  // The Operand indexing map may contain "zero" results, e.g.:
+  //    (d0, d1, d2, d3) -> (d0, d1, d2, 0)
+  // When applied to canonical vector shapes like these:
+  //    (1, 16, 16, 4)
+  // we would get:
+  //    (1, 16, 16, 0)
+  // Instead, we should extract the following map permutation map for masking:
+  //    (d0, d1, d2, d3) -> (d0, d1, d2)
+  // This way, the corresponding vector/mask type will be:
+  //    vector<1x16x16xty>
+  // rather than:
+  //    vector<1x16x16x0xty>
+  if (maybeIndexingMap)
+    maybeMaskingMap = maybeIndexingMap->dropZeroResults();
+
   // Create or retrieve mask for this operation.
   Value mask =
       getOrCreateMaskFor(rewriter, opToMask, linalgOp, maybeMaskingMap);
@@ -476,7 +488,8 @@ static AffineMap reindexIndexingMap(AffineMap map) {
   assert(map.isProjectedPermutation(/*allowZeroInResults=*/true) &&
          "expected projected permutation");
   auto res = compressUnusedDims(map);
-  assert(res.getNumDims() == res.getNumResults() &&
+  assert(res.getNumDims() ==
+             (res.getNumResults() - res.getNumOfZeroResults()) &&
          "expected reindexed map with same number of dims and results");
   return res;
 }
@@ -1349,16 +1362,6 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     // permutation map and masking map.
     AffineMap indexingMap = linalgOp.getMatchingIndexingMap(opOperand);
 
-    // Remove zeros from indexing map to use it as masking map.
-    SmallVector<int64_t> zeroPos;
-    auto results = indexingMap.getResults();
-    for (const auto &result : llvm::enumerate(results)) {
-      if (isa<AffineConstantExpr>(result.value())) {
-        zeroPos.push_back(result.index());
-      }
-    }
-    AffineMap maskingMap = indexingMap.dropResults(zeroPos);
-
     AffineMap readMap;
     VectorType readType;
     Type elemType = getElementTypeOrSelf(opOperand->get());
@@ -1388,7 +1391,7 @@ vectorizeAsLinalgGeneric(RewriterBase &rewriter, VectorizationState &state,
     Operation *read = rewriter.create<vector::TransferReadOp>(
         loc, readType, opOperand->get(), indices, readMap,
         ArrayRef<bool>(inBounds));
-    read = state.maskOperation(rewriter, read, linalgOp, maskingMap);
+    read = state.maskOperation(rewriter, read, linalgOp, indexingMap);
     Value readValue = read->getResult(0);
 
     // 3.b. If masked, set in-bounds to true. Masking guarantees that the access
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 6a3f2fc5fbc49..38e427af1c484 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -249,41 +249,6 @@ Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
   return sliceOp.getSource();
 }
 
-GenericOp makeTransposeOp(OpBuilder &b, Location loc, Value inputTensor,
-                          Value outputTensor,
-                          ArrayRef<int64_t> transposeVector) {
-  auto resultTensorType = cast<RankedTensorType>(outputTensor.getType());
-  Type elementType = resultTensorType.getElementType();
-
-  assert(isPermutationVector(transposeVector) &&
-         "expect transpose vector to be a permutation");
-  assert(transposeVector.size() ==
-             static_cast<size_t>(resultTensorType.getRank()) &&
-         "expect transpose vector size to match result tensor rank");
-
-  // Compute the transpose and the indentity indexing maps.
-  SmallVector<AffineMap> indexingMaps = {
-      inversePermutation(AffineMap::getPermutationMap(
-          SmallVector<unsigned>(transposeVector), b.getContext())),
-      AffineMap::getMultiDimIdentityMap(transposeVector.size(),
-                                        b.getContext())};
-  SmallVector<utils::IteratorType> iteratorTypes(transposeVector.size(),
-                                                 utils::IteratorType::parallel);
-
-  // Create a GenericOp to transpose `inputTensor` into `outputTensor`.
-  auto transposeOp =
-      b.create<GenericOp>(loc, resultTensorType, inputTensor, outputTensor,
-                          indexingMaps, iteratorTypes);
-
-  // Create the body of the transpose operation.
-  OpBuilder::InsertionGuard g(b);
-  Region &body = transposeOp.getRegion();
-  Block *bodyBlock = b.createBlock(&body, /*insertPt=*/{},
-                                   {elementType, elementType}, {loc, loc});
-  b.create<YieldOp>(loc, bodyBlock->getArgument(0));
-  return transposeOp;
-}
-
 GenericOp makeMemRefCopyOp(OpBuilder &b, Location loc, Value from, Value to) {
   auto memrefTypeTo = cast<MemRefType>(to.getType());
 #ifndef NDEBUG
diff --git a/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
index 2a5b4fbcb5271..e1c0c2410c126 100644
--- a/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Math/Transforms/CMakeLists.txt
@@ -1,7 +1,7 @@
 add_mlir_dialect_library(MLIRMathTransforms
   AlgebraicSimplification.cpp
   ExpandPatterns.cpp
-  LegalizeToF32.cpp
+  ExtendToSupportedTypes.cpp
   PolynomialApproximation.cpp
   UpliftToFMA.cpp
 
diff --git a/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
new file mode 100644
index 0000000000000..1a9eafec9fdd5
--- /dev/null
+++ b/mlir/lib/Dialect/Math/Transforms/ExtendToSupportedTypes.cpp
@@ -0,0 +1,164 @@
+//===- ExtendToSupportedTypes.cpp - Legalize functions on unsupported floats
+//----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements legalizing math operations on unsupported floating-point
+// types through arith.extf and arith.truncf.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
+#include "mlir/Dialect/Math/IR/Math.h"
+#include "mlir/Dialect/Math/Transforms/Passes.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
+
+namespace mlir::math {
+#define GEN_PASS_DEF_MATHEXTENDTOSUPPORTEDTYPES
+#include "mlir/Dialect/Math/Transforms/Passes.h.inc"
+} // namespace mlir::math
+
+using namespace mlir;
+
+namespace {
+struct ExtendToSupportedTypesRewritePattern final : ConversionPattern {
+  ExtendToSupportedTypesRewritePattern(TypeConverter &converter,
+                                       MLIRContext *context)
+      : ConversionPattern(converter, MatchAnyOpTypeTag{}, 1, context) {}
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
+struct ExtendToSupportedTypesPass
+    : mlir::math::impl::MathExtendToSupportedTypesBase<
+          ExtendToSupportedTypesPass> {
+  using math::impl::MathExtendToSupportedTypesBase<
+      ExtendToSupportedTypesPass>::MathExtendToSupportedTypesBase;
+
+  void runOnOperation() override;
+};
+} // namespace
+
+void mlir::math::populateExtendToSupportedTypesTypeConverter(
+    TypeConverter &typeConverter, const SetVector<Type> &sourceTypes,
+    Type targetType) {
+
+  typeConverter.addConversion(
+      [](Type type) -> std::optional<Type> { return type; });
+  typeConverter.addConversion(
+      [&sourceTypes, targetType](FloatType type) -> std::optional<Type> {
+        if (!sourceTypes.contains(type))
+          return targetType;
+
+        return std::nullopt;
+      });
+  typeConverter.addConversion(
+      [&sourceTypes, targetType](ShapedType type) -> std::optional<Type> {
+        if (auto elemTy = dyn_cast<FloatType>(type.getElementType()))
+          if (!sourceTypes.contains(elemTy))
+            return type.clone(targetType);
+
+        return std::nullopt;
+      });
+  typeConverter.addTargetMaterialization(
+      [](OpBuilder &b, Type target, ValueRange input, Location loc) {
+        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
+        extFOp.setFastmath(arith::FastMathFlags::contract);
+        return extFOp;
+      });
+}
+
+void mlir::math::populateExtendToSupportedTypesConversionTarget(
+    ConversionTarget &target, TypeConverter &typeConverter) {
+  target.markUnknownOpDynamicallyLegal([&typeConverter](Operation *op) -> bool {
+    if (isa<MathDialect>(op->getDialect()))
+      return typeConverter.isLegal(op);
+    return true;
+  });
+  target.addLegalOp<FmaOp>();
+  target.addLegalOp<arith::ExtFOp, arith::TruncFOp>();
+}
+
+LogicalResult ExtendToSupportedTypesRewritePattern::matchAndRewrite(
+    Operation *op, ArrayRef<Value> operands,
+    ConversionPatternRewriter &rewriter) const {
+  Location loc = op->getLoc();
+  const TypeConverter *converter = getTypeConverter();
+  FailureOr<Operation *> legalized =
+      convertOpResultTypes(op, operands, *converter, rewriter);
+  if (failed(legalized))
+    return failure();
+
+  SmallVector<Value> results = (*legalized)->getResults();
+  for (auto [result, newType, origType] : llvm::zip_equal(
+           results, (*legalized)->getResultTypes(), op->getResultTypes())) {
+    if (newType != origType) {
+      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
+      truncFOp.setFastmath(arith::FastMathFlags::contract);
+      result = truncFOp.getResult();
+    }
+  }
+  rewriter.replaceOp(op, results);
+  return success();
+}
+
+void mlir::math::populateExtendToSupportedTypesPatterns(
+    RewritePatternSet &patterns, TypeConverter &typeConverter) {
+  patterns.add<ExtendToSupportedTypesRewritePattern>(typeConverter,
+                                                     patterns.getContext());
+}
+
+void ExtendToSupportedTypesPass::runOnOperation() {
+  Operation *op = getOperation();
+  MLIRContext *ctx = &getContext();
+
+  // Parse target type
+  std::optional<Type> maybeTargetType =
+      arith::parseFloatType(ctx, targetTypeStr);
+  if (!maybeTargetType.has_value()) {
+    emitError(UnknownLoc::get(ctx), "could not map target type '" +
+                                        targetTypeStr +
+                                        "' to a known floating-point type");
+    return signalPassFailure();
+  }
+  Type targetType = maybeTargetType.value();
+
+  // Parse source types
+  llvm::SetVector<Type> sourceTypes;
+  for (const auto &extraTypeStr : extraTypeStrs) {
+    std::optional<FloatType> maybeExtraType =
+        arith::parseFloatType(ctx, extraTypeStr);
+    if (!maybeExtraType.has_value()) {
+      emitError(UnknownLoc::get(ctx), "could not map source type '" +
+                                          extraTypeStr +
+                                          "' to a known floating-point type");
+      return signalPassFailure();
+    }
+    sourceTypes.insert(maybeExtraType.value());
+  }
+  // f64 and f32 are implicitly supported
+  Builder b(ctx);
+  sourceTypes.insert(b.getF64Type());
+  sourceTypes.insert(b.getF32Type());
+
+  TypeConverter typeConverter;
+  math::populateExtendToSupportedTypesTypeConverter(typeConverter, sourceTypes,
+                                                    targetType);
+  ConversionTarget target(*ctx);
+  math::populateExtendToSupportedTypesConversionTarget(target, typeConverter);
+  RewritePatternSet patterns(ctx);
+  math::populateExtendToSupportedTypesPatterns(patterns, typeConverter);
+  if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    return signalPassFailure();
+}
diff --git a/mlir/lib/Dialect/Math/Transforms/LegalizeToF32.cpp b/mlir/lib/Dialect/Math/Transforms/LegalizeToF32.cpp
deleted file mode 100644
index 2e60fe455dcad..0000000000000
--- a/mlir/lib/Dialect/Math/Transforms/LegalizeToF32.cpp
+++ /dev/null
@@ -1,118 +0,0 @@
-//===- LegalizeToF32.cpp - Legalize functions on small floats ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements legalizing math operations on small floating-point
-// types through arith.extf and arith.truncf.
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Dialect/Arith/IR/Arith.h"
-#include "mlir/Dialect/Math/IR/Math.h"
-#include "mlir/Dialect/Math/Transforms/Passes.h"
-#include "mlir/IR/Diagnostics.h"
-#include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/TypeUtilities.h"
-#include "mlir/Transforms/DialectConversion.h"
-#include "llvm/ADT/STLExtras.h"
-
-namespace mlir::math {
-#define GEN_PASS_DEF_MATHLEGALIZETOF32
-#include "mlir/Dialect/Math/Transforms/Passes.h.inc"
-} // namespace mlir::math
-
-using namespace mlir;
-namespace {
-struct LegalizeToF32RewritePattern final : ConversionPattern {
-  LegalizeToF32RewritePattern(TypeConverter &converter, MLIRContext *context)
-      : ConversionPattern(converter, MatchAnyOpTypeTag{}, 1, context) {}
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override;
-};
-
-struct LegalizeToF32Pass final
-    : mlir::math::impl::MathLegalizeToF32Base<LegalizeToF32Pass> {
-  void runOnOperation() override;
-};
-} // namespace
-
-void mlir::math::populateLegalizeToF32TypeConverter(
-    TypeConverter &typeConverter) {
-  typeConverter.addConversion(
-      [](Type type) -> std::optional<Type> { return type; });
-  typeConverter.addConversion([](FloatType type) -> std::optional<Type> {
-    if (type.getWidth() < 32)
-      return Float32Type::get(type.getContext());
-    return std::nullopt;
-  });
-  typeConverter.addConversion([](ShapedType type) -> std::optional<Type> {
-    if (auto elemTy = dyn_cast<FloatType>(type.getElementType()))
-      return type.clone(Float32Type::get(type.getContext()));
-    return std::nullopt;
-  });
-  typeConverter.addTargetMaterialization(
-      [](OpBuilder &b, Type target, ValueRange input, Location loc) {
-        auto extFOp = b.create<arith::ExtFOp>(loc, target, input);
-        extFOp.setFastmath(arith::FastMathFlags::contract);
-        return extFOp;
-      });
-}
-
-void mlir::math::populateLegalizeToF32ConversionTarget(
-    ConversionTarget &target, TypeConverter &typeConverter) {
-  target.markUnknownOpDynamicallyLegal([&typeConverter](Operation *op) -> bool {
-    if (isa<MathDialect>(op->getDialect()))
-      return typeConverter.isLegal(op);
-    return true;
-  });
-  target.addLegalOp<FmaOp>();
-  target.addLegalOp<arith::ExtFOp, arith::TruncFOp>();
-}
-
-LogicalResult LegalizeToF32RewritePattern::matchAndRewrite(
-    Operation *op, ArrayRef<Value> operands,
-    ConversionPatternRewriter &rewriter) const {
-  Location loc = op->getLoc();
-  const TypeConverter *converter = getTypeConverter();
-  FailureOr<Operation *> legalized =
-      convertOpResultTypes(op, operands, *converter, rewriter);
-  if (failed(legalized))
-    return failure();
-
-  SmallVector<Value> results = (*legalized)->getResults();
-  for (auto [result, newType, origType] : llvm::zip_equal(
-           results, (*legalized)->getResultTypes(), op->getResultTypes())) {
-    if (newType != origType) {
-      auto truncFOp = rewriter.create<arith::TruncFOp>(loc, origType, result);
-      truncFOp.setFastmath(arith::FastMathFlags::contract);
-      result = truncFOp.getResult();
-    }
-  }
-  rewriter.replaceOp(op, results);
-  return success();
-}
-
-void mlir::math::populateLegalizeToF32Patterns(RewritePatternSet &patterns,
-                                               TypeConverter &typeConverter) {
-  patterns.add<LegalizeToF32RewritePattern>(typeConverter,
-                                            patterns.getContext());
-}
-
-void LegalizeToF32Pass::runOnOperation() {
-  Operation *op = getOperation();
-  MLIRContext &ctx = getContext();
-
-  TypeConverter typeConverter;
-  math::populateLegalizeToF32TypeConverter(typeConverter);
-  ConversionTarget target(ctx);
-  math::populateLegalizeToF32ConversionTarget(target, typeConverter);
-  RewritePatternSet patterns(&ctx);
-  math::populateLegalizeToF32Patterns(patterns, typeConverter);
-  if (failed(applyPartialConversion(op, target, std::move(patterns))))
-    return signalPassFailure();
-}
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
index 0cb5931ce6bf9..fb2921fec9f79 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ResolveShapedTypeResultDims.cpp
@@ -103,6 +103,44 @@ struct DimOfReifyRankedShapedTypeOpInterface : public OpRewritePattern<OpTy> {
     return success();
   }
 };
+
+/// Fold dim ops of iter_args to dim ops of their respective init args. E.g.:
+///
+/// ```
+/// %0 = ... : tensor<?x?xf32>
+/// scf.forall ... shared_outs(%arg0 = %0) -> (tensor<?x?xf32>) {
+///   %1 = tensor.dim %arg0, %c0 : tensor<?x?xf32>
+///   ...
+/// }
+/// ```
+///
+/// is folded to:
+///
+/// ```
+/// %0 = ... : tensor<?x?xf32>
+/// scf.forall ... shared_outs(%arg0 = %0) -> (tensor<?x?xf32>) {
+///   %1 = tensor.dim %0, %c0 : tensor<?x?xf32>
+///   ...
+/// }
+/// ```
+struct IterArgsToInitArgs : public OpRewritePattern<tensor::DimOp> {
+  using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(tensor::DimOp dimOp,
+                                PatternRewriter &rewriter) const final {
+    auto blockArg = dyn_cast<BlockArgument>(dimOp.getSource());
+    if (!blockArg)
+      return failure();
+    auto loopLikeOp =
+        dyn_cast<LoopLikeOpInterface>(blockArg.getParentBlock()->getParentOp());
+    if (!loopLikeOp)
+      return failure();
+    Value initArg = loopLikeOp.getTiedLoopInit(blockArg)->get();
+    rewriter.modifyOpInPlace(
+        dimOp, [&]() { dimOp.getSourceMutable().assign(initArg); });
+    return success();
+  }
+};
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -127,8 +165,8 @@ struct ResolveShapedTypeResultDimsPass final
 void memref::populateResolveRankedShapedTypeResultDimsPatterns(
     RewritePatternSet &patterns) {
   patterns.add<DimOfReifyRankedShapedTypeOpInterface<memref::DimOp>,
-               DimOfReifyRankedShapedTypeOpInterface<tensor::DimOp>>(
-      patterns.getContext());
+               DimOfReifyRankedShapedTypeOpInterface<tensor::DimOp>,
+               IterArgsToInitArgs>(patterns.getContext());
 }
 
 void memref::populateResolveShapedTypeResultDimsPatterns(
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index db47276dcefe9..90bf5df67b03b 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1225,17 +1225,13 @@ parsePrivateList(OpAsmParser &parser,
 }
 
 static void printPrivateList(OpAsmPrinter &p, Operation *op,
-                             ValueRange privateVars, TypeRange privateTypes,
-                             ArrayAttr privateSyms) {
-  // TODO: Remove target-specific logic from this function.
-  auto targetOp = mlir::dyn_cast<mlir::omp::TargetOp>(op);
-  assert(targetOp);
-
+                             Operation::operand_range privateVars,
+                             TypeRange privateTypes, ArrayAttr privateSyms) {
   auto &region = op->getRegion(0);
   auto *argsBegin = region.front().getArguments().begin();
-  MutableArrayRef argsSubrange(argsBegin + targetOp.getMapVars().size(),
-                               argsBegin + targetOp.getMapVars().size() +
-                                   privateTypes.size());
+  MutableArrayRef argsSubrange(argsBegin + privateVars.getBeginOperandIndex(),
+                               argsBegin + privateVars.getBeginOperandIndex() +
+                                   privateVars.size());
   mlir::SmallVector<bool> isByRefVec;
   isByRefVec.resize(privateTypes.size(), false);
   DenseBoolArrayAttr isByRef =
@@ -1859,11 +1855,11 @@ LogicalResult SimdOp::verify() {
 
 void DistributeOp::build(OpBuilder &builder, OperationState &state,
                          const DistributeOperands &clauses) {
-  // TODO Store clauses in op: privateVars, privateSyms.
-  DistributeOp::build(
-      builder, state, clauses.allocateVars, clauses.allocatorVars,
-      clauses.distScheduleStatic, clauses.distScheduleChunkSize, clauses.order,
-      clauses.orderMod, /*private_vars=*/{}, /*private_syms=*/nullptr);
+  DistributeOp::build(builder, state, clauses.allocateVars,
+                      clauses.allocatorVars, clauses.distScheduleStatic,
+                      clauses.distScheduleChunkSize, clauses.order,
+                      clauses.orderMod, clauses.privateVars,
+                      makeArrayAttr(builder.getContext(), clauses.privateSyms));
 }
 
 LogicalResult DistributeOp::verify() {
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index 9d7c36520874d..d3f7c9798b9b8 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -387,7 +387,7 @@ LogicalResult PatternOp::verifyRegions() {
 void PatternOp::build(OpBuilder &builder, OperationState &state,
                       std::optional<uint16_t> benefit,
                       std::optional<StringRef> name) {
-  build(builder, state, builder.getI16IntegerAttr(benefit ? *benefit : 0),
+  build(builder, state, builder.getI16IntegerAttr(benefit.value_or(0)),
         name ? builder.getStringAttr(*name) : StringAttr());
   state.regions[0]->emplaceBlock();
 }
diff --git a/mlir/lib/Dialect/Quant/CMakeLists.txt b/mlir/lib/Dialect/Quant/CMakeLists.txt
index 037bba8dcb5c9..31167e6af908b 100644
--- a/mlir/lib/Dialect/Quant/CMakeLists.txt
+++ b/mlir/lib/Dialect/Quant/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(IR)
+add_subdirectory(Transforms)
 add_subdirectory(Utils)
diff --git a/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.cpp b/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.cpp
index c0c00fb4893cb..6a4ac310eb052 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.cpp
@@ -9,8 +9,8 @@
 
 #include "QuantDialectBytecode.h"
 #include "mlir/Bytecode/BytecodeImplementation.h"
-#include "mlir/Dialect/Quant/QuantOps.h"
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallVector.h"
@@ -31,7 +31,7 @@ static LogicalResult readDoubleAPFloat(DialectBytecodeReader &reader,
   return success();
 }
 
-#include "mlir/Dialect/Quant/QuantDialectBytecode.cpp.inc"
+#include "mlir/Dialect/Quant/IR/QuantDialectBytecode.cpp.inc"
 
 /// This class implements the bytecode interface for the Quant dialect.
 struct QuantDialectBytecodeInterface : public BytecodeDialectInterface {
@@ -64,6 +64,6 @@ struct QuantDialectBytecodeInterface : public BytecodeDialectInterface {
 };
 } // namespace
 
-void quant::detail::addBytecodeInterface(QuantizationDialect *dialect) {
+void quant::detail::addBytecodeInterface(QuantDialect *dialect) {
   dialect->addInterfaces<QuantDialectBytecodeInterface>();
 }
diff --git a/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.h b/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.h
index 9e9cbf66d84d9..eef2b5bbefecc 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.h
+++ b/mlir/lib/Dialect/Quant/IR/QuantDialectBytecode.h
@@ -15,12 +15,12 @@
 #define LIB_MLIR_DIALECT_QUANT_IR_QUANTDIALECTBYTECODE_H
 
 namespace mlir::quant {
-class QuantizationDialect;
+class QuantDialect;
 
 namespace detail {
 /// Add the interfaces necessary for encoding the quantization dialect
 /// components in bytecode.
-void addBytecodeInterface(QuantizationDialect *dialect);
+void addBytecodeInterface(QuantDialect *dialect);
 } // namespace detail
 } // namespace mlir::quant
 
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index c9a6bbc9ceeea..c584903f3a15d 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -6,44 +6,209 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/QuantOps.h"
 #include "QuantDialectBytecode.h"
 #include "TypeDetail.h"
 
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
-#include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/MathExtras.h"
-#include <numeric>
+#include "mlir/IR/TypeUtilities.h"
 
-using namespace mlir;
-using namespace mlir::quant;
-using namespace mlir::quant::detail;
+#include "mlir/Dialect/Quant/IR/QuantOpsDialect.cpp.inc"
 
-#include "mlir/Dialect/Quant/QuantOpsDialect.cpp.inc"
 
-void QuantizationDialect::initialize() {
+namespace mlir {
+namespace quant {
+
+namespace {
+
+// Verify the integrity of per-axis quantization information, if present.
+//
+// - quantizedType
+//   Any quantized type. Any quantized type with no per-axis quantization is
+//   ignored.
+//
+// - containerType
+//   Original input or result type of the operation using the provided quantized
+//   type. Used to ensure that the quantized type appears within a tensor and
+//   that the tensor is compatible with per-axis quantization information.
+//
+LogicalResult verifyPerAxisQuantization(Operation *op,
+                                        QuantizedType quantizedType,
+                                        Type containerType) {
+  auto quantizedPerAxisType = dyn_cast<UniformQuantizedPerAxisType>(quantizedType);
+  if (!quantizedPerAxisType)
+    return success();
+
+  auto tensorType = dyn_cast<TensorType>(containerType);
+  if (!tensorType)
+    return op->emitError("scalar types may not use per-axis quantization");
+
+  if (!tensorType.hasRank())
+    return success();
+
+  int64_t quantizedDimension = quantizedPerAxisType.getQuantizedDimension();
+  if (quantizedDimension >= tensorType.getRank())
+    return op->emitError("quantized dimension must be less than tensor rank");
+
+  int64_t quantizedDimensionSize = tensorType.getDimSize(quantizedDimension);
+  if (quantizedDimensionSize != ShapedType::kDynamic &&
+      quantizedDimensionSize != (int64_t)quantizedPerAxisType.getScales().size())
+    return op->emitError(
+        "quantized dimension size does not match number of scales");
+
+  return success();
+}
+
+// Common verification logic for 'quant.dcast' and 'quant.qcast' ops.
+//
+// - quantizedType
+//   Quantized type used in the input ('quant.dcast') or result ('quant.qcast'),
+//   whether as a primitive type or in a tensor.
+//
+// - floatType
+//   Float type used in the input ('quant.qcast') or result ('quant.dcast'),
+//   whether as a primitive type or in a tensor.
+//
+// - containerType
+//   Type of original input or result.
+//
+LogicalResult verifyQuantizationOp(Operation *op, QuantizedType quantizedType,
+                                   FloatType floatType, Type containerType) {
+  if (quantizedType.getExpressedType() != floatType)
+    return op->emitError(
+        "expressed type in quantized type expected to match float type");
+
+  // Veriy integrity of per-axis quantization information, if present.
+  return verifyPerAxisQuantization(op, quantizedType, containerType);
+}
+
+}  // namespace
+
+
+//===----------------------------------------------------------------------===//
+// Dialect
+//===----------------------------------------------------------------------===//
+
+void QuantDialect::initialize() {
   addTypes<AnyQuantizedType, CalibratedQuantizedType, UniformQuantizedType,
            UniformQuantizedPerAxisType>();
   addOperations<
 #define GET_OP_LIST
-#include "mlir/Dialect/Quant/QuantOps.cpp.inc"
+#include "mlir/Dialect/Quant/IR/QuantOps.cpp.inc"
       >();
-  addBytecodeInterface(this);
+  detail::addBytecodeInterface(this);
+}
+
+
+//===----------------------------------------------------------------------===//
+// DequantizeCastOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult DequantizeCastOp::verify() {
+  return verifyQuantizationOp(*this, getQuantizedType(), getFloatType(),
+                              getInput().getType());
+}
+
+OpFoldResult DequantizeCastOp::fold(FoldAdaptor adaptor) {
+  // Matches x -> quant.qcast -> quant.dcast -> y, replacing the quant.dcast op
+  // with the value of x. Values x and y are guaranteed to be of the same type
+  // in this pattern.
+  auto srcQcastOp = getInput().getDefiningOp<QuantizeCastOp>();
+  if (!srcQcastOp)
+    return {};
+  assert(srcQcastOp.getInput().getType() == getType());
+  return srcQcastOp.getInput();
+}
+
+FloatType DequantizeCastOp::getFloatType() {
+  return cast<FloatType>(getElementTypeOrSelf(getResult().getType()));
+}
+
+QuantizedType DequantizeCastOp::getQuantizedType() {
+  return cast<QuantizedType>(getElementTypeOrSelf(getInput().getType()));
+}
+
+
+//===----------------------------------------------------------------------===//
+// QuantizeCastOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult QuantizeCastOp::verify() {
+  return verifyQuantizationOp(*this, getQuantizedType(), getFloatType(),
+                              getInput().getType());
+}
+
+OpFoldResult QuantizeCastOp::fold(FoldAdaptor adaptor) {
+  // Matches x -> quant.dcast -> quant.qcast -> y, replacing the quant.qcast op
+  // with the value of x if the casts invert each other. Contrary to the folding
+  // pattern in quant.dcast (i.e., x -> quant.qcast -> quant.dcast -> y), values
+  // x and y are not guaranteed to be of the same type here, as they may use
+  // different quantization parameters.
+  auto srcDcastOp = getInput().getDefiningOp<DequantizeCastOp>();
+  if (!srcDcastOp || srcDcastOp.getInput().getType() != getType())
+    return {};
+  return srcDcastOp.getInput();
+}
+
+FloatType QuantizeCastOp::getFloatType() {
+  return cast<FloatType>(getElementTypeOrSelf(getInput().getType()));
+}
+
+QuantizedType QuantizeCastOp::getQuantizedType() {
+  return cast<QuantizedType>(getElementTypeOrSelf(getResult().getType()));
+}
+
+
+//===----------------------------------------------------------------------===//
+// StorageCastOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult StorageCastOp::verify() {
+  auto quantizedType = getQuantizedType();
+  auto integerType = getIntegerType();
+  if (quantizedType.getStorageType() != integerType)
+    return emitError(
+        "storage type in quantized type expected to match integer type");
+
+  // Verify integrity of per-axis quantization information, if available. While
+  // the quantization type may appear in the input or the result, their tensor
+  // shapes are guaranteed to be identical at this point.
+  return verifyPerAxisQuantization(*this, quantizedType, getInput().getType());
 }
 
 OpFoldResult StorageCastOp::fold(FoldAdaptor adaptor) {
-  // Matches x -> [scast -> scast] -> y, replacing the second scast with the
-  // value of x if the casts invert each other.
-  auto srcScastOp = getArg().getDefiningOp<StorageCastOp>();
-  if (!srcScastOp || srcScastOp.getArg().getType() != getType())
-    return OpFoldResult();
-  return srcScastOp.getArg();
+  // Matches x -> quant.scast -> quant.scast -> y, replacing the second
+  // quant.scast with the value of x if the casts invert each other.
+  auto srcScastOp = getInput().getDefiningOp<StorageCastOp>();
+  if (!srcScastOp || srcScastOp.getInput().getType() != getType())
+    return {};
+  return srcScastOp.getInput();
+}
+
+IntegerType StorageCastOp::getIntegerType() {
+  auto inputScalarType = getElementTypeOrSelf(getInput().getType());
+  if (auto integerType = dyn_cast<IntegerType>(inputScalarType))
+    return integerType;
+
+  auto resultScalarType = getElementTypeOrSelf(getResult().getType());
+  return cast<IntegerType>(resultScalarType);
+}
+
+QuantizedType StorageCastOp::getQuantizedType() {
+  auto inputScalarType = getElementTypeOrSelf(getInput().getType());
+  if (auto quantizedType = dyn_cast<QuantizedType>(inputScalarType))
+    return quantizedType;
+
+  auto resultScalarType = getElementTypeOrSelf(getResult().getType());
+  return cast<QuantizedType>(resultScalarType);
 }
 
+
+} // namespace quant
+} // namespace mlir
+
 #define GET_OP_CLASSES
-#include "mlir/Dialect/Quant/QuantOps.cpp.inc"
+#include "mlir/Dialect/Quant/IR/QuantOps.cpp.inc"
+
diff --git a/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp b/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
index c2ba9c04e8771..ac01b37a55307 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/QuantTypes.h"
 #include "TypeDetail.h"
-#include "mlir/Dialect/Quant/QuantOps.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
@@ -20,12 +20,28 @@ using namespace mlir;
 using namespace mlir::quant;
 using namespace mlir::quant::detail;
 
+namespace {
+
+// Return the minimum scale representable in a given float type
+double getMinScale(Type expressedType) {
+  auto floatType = cast<FloatType>(expressedType);
+  return APFloat::getSmallest(floatType.getFloatSemantics()).convertToDouble();
+}
+
+// Return the maximum scale representable in a given float type
+double getMaxScale(Type expressedType) {
+  auto floatType = cast<FloatType>(expressedType);
+  return APFloat::getLargest(floatType.getFloatSemantics()).convertToDouble();
+}
+
+}  // namespace
+
 unsigned QuantizedType::getFlags() const {
   return static_cast<ImplType *>(impl)->flags;
 }
 
 bool QuantizedType::classof(Type type) {
-  return llvm::isa<QuantizationDialect>(type.getDialect());
+  return llvm::isa<QuantDialect>(type.getDialect());
 }
 
 LogicalResult
@@ -73,6 +89,17 @@ int64_t QuantizedType::getStorageTypeMax() const {
   return static_cast<ImplType *>(impl)->storageTypeMax;
 }
 
+bool QuantizedType::hasStorageTypeBounds() const {
+  unsigned int integralWidth = getStorageTypeIntegralWidth();
+  bool isSignedInteger = isSigned();
+  int64_t defaultIntegerMin =
+      getDefaultMinimumForInteger(isSignedInteger, integralWidth);
+  int64_t defaultIntegerMax =
+      getDefaultMaximumForInteger(isSignedInteger, integralWidth);
+  return defaultIntegerMin != getStorageTypeMin() ||
+         defaultIntegerMax != getStorageTypeMax();
+}
+
 unsigned QuantizedType::getStorageTypeIntegralWidth() const {
   // NOTE: If ever supporting non-integral storage types, some other scheme
   // for determining the width will be needed.
@@ -293,8 +320,13 @@ LogicalResult UniformQuantizedType::verifyInvariants(
     return emitError() << "expressed type must be floating point";
 
   // Verify scale.
+  double minScale = getMinScale(expressedType);
+  double maxScale = getMaxScale(expressedType);
   if (scale <= 0.0 || std::isinf(scale) || std::isnan(scale))
     return emitError() << "illegal scale: " << scale;
+  if (scale < minScale || scale > maxScale)
+    return emitError() << "scale out of expressed type range [" << minScale
+                       << ", " << maxScale << "]";
 
   return success();
 }
@@ -353,11 +385,20 @@ LogicalResult UniformQuantizedPerAxisType::verifyInvariants(
                        << scales.size() << ", " << zeroPoints.size();
 
   // Verify scale.
+  double minScale = getMinScale(expressedType);
+  double maxScale = getMaxScale(expressedType);
   for (double scale : scales) {
     if (scale <= 0.0 || std::isinf(scale) || std::isnan(scale))
       return emitError() << "illegal scale: " << scale;
+    if (scale < minScale || scale > maxScale)
+      return emitError() << "scale out of expressed type range [" << minScale
+                         << ", " << maxScale << "]";
   }
 
+  // Verify quantized dimension.
+  if (quantizedDimension < 0)
+    return emitError() << "illegal quantized dimension: " << quantizedDimension;
+
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Quant/IR/TypeParser.cpp b/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
index 926a8a0aa13d5..851763d8942e8 100644
--- a/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
+++ b/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/QuantOps.h"
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Location.h"
@@ -317,7 +317,7 @@ static Type parseCalibratedType(DialectAsmParser &parser) {
 }
 
 /// Parse a type registered to this dialect.
-Type QuantizationDialect::parseType(DialectAsmParser &parser) const {
+Type QuantDialect::parseType(DialectAsmParser &parser) const {
   // All types start with an identifier that we switch on.
   StringRef typeNameSpelling;
   if (failed(parser.parseKeyword(&typeNameSpelling)))
@@ -346,12 +346,7 @@ static void printStorageType(QuantizedType type, DialectAsmPrinter &out) {
   }
 
   // storageTypeMin and storageTypeMax if not default.
-  int64_t defaultIntegerMin =
-      QuantizedType::getDefaultMinimumForInteger(isSigned, storageWidth);
-  int64_t defaultIntegerMax =
-      QuantizedType::getDefaultMaximumForInteger(isSigned, storageWidth);
-  if (defaultIntegerMin != type.getStorageTypeMin() ||
-      defaultIntegerMax != type.getStorageTypeMax()) {
+  if (type.hasStorageTypeBounds()) {
     out << "<" << type.getStorageTypeMin() << ":" << type.getStorageTypeMax()
         << ">";
   }
@@ -419,7 +414,7 @@ static void printCalibratedQuantizedType(CalibratedQuantizedType type,
 }
 
 /// Print a type registered to this dialect.
-void QuantizationDialect::printType(Type type, DialectAsmPrinter &os) const {
+void QuantDialect::printType(Type type, DialectAsmPrinter &os) const {
   if (auto anyType = llvm::dyn_cast<AnyQuantizedType>(type))
     printAnyQuantizedType(anyType, os);
   else if (auto uniformType = llvm::dyn_cast<UniformQuantizedType>(type))
diff --git a/mlir/lib/Dialect/Quant/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Quant/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..2fd4a41999d45
--- /dev/null
+++ b/mlir/lib/Dialect/Quant/Transforms/CMakeLists.txt
@@ -0,0 +1,26 @@
+add_mlir_dialect_library(MLIRQuantTransforms
+  LowerQuantOps.cpp
+  StripFuncQuantTypes.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  {$MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Quant/Transforms
+
+  DEPENDS
+  MLIRQuantTransformsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRArithDialect
+  MLIRFuncDialect
+  MLIRFuncTransforms
+  MLIRIndexDialect
+  MLIRIR
+  MLIRLinalgDialect
+  MLIRLinalgUtils
+  MLIRPass
+  MLIRQuantDialect
+  MLIRShapeDialect
+  MLIRTensorDialect
+  MLIRTransforms
+  MLIRTransformUtils
+
+  )
diff --git a/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
new file mode 100644
index 0000000000000..4adeb9218ff8e
--- /dev/null
+++ b/mlir/lib/Dialect/Quant/Transforms/LowerQuantOps.cpp
@@ -0,0 +1,676 @@
+//===- LowerQuantOps.cpp - Lower 'quant' dialect ops ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Transforms `quant.dcast` and `quant.qcast` into lower-level ops.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
+#include "mlir/Dialect/Quant/Transforms/Passes.h"
+#include "mlir/Dialect/Shape/IR/Shape.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/Matchers.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+namespace quant {
+
+#define GEN_PASS_DEF_LOWERQUANTOPS
+#include "mlir/Dialect/Quant/Transforms/Passes.h.inc"
+
+namespace {
+
+// If 'inputType' is a tensor, return its element type. If it is a scalar,
+// return it as is.
+Type getScalarType(Type inputType) {
+  if (auto tensorType = dyn_cast<TensorType>(inputType))
+    return tensorType.getElementType();
+  return inputType;
+}
+
+// Return the shape of an input value as a list of attributes (static dimensions)
+// and values (dynamic dimensions). If 'input' is a scalar, an empty list is
+// returned. If 'input' is a tensor, its shape is returned.
+SmallVector<OpFoldResult>
+getScalarOrTensorShape(OpBuilder &builder, Location loc, Value input) {
+  if (isa<TensorType>(input.getType()))
+    return tensor::getMixedSizes(builder, loc, input);
+  return {};
+}
+
+// If 'referenceType' is a scalar, return 'elementType' as is. If
+// 'referenceType' is a tensor, return another tensor with the same shape and
+// elements of type 'elementType'.
+Type getScalarOrTensorType(Type elementType, Type referenceType) {
+  if (auto tensorType = dyn_cast<TensorType>(referenceType))
+    return tensorType.clone(elementType);
+  return elementType;
+}
+
+// Return a constant with the given value. If 'referenceType' is a tensor, a
+// tensor splat of shape 'referenceShape' is returned. If 'referenceType' is a
+// scalar, 'referenceShape' is ignored and a scalar constant is returned.
+Value getScalarOrTensorConstant(OpBuilder &builder, Location loc, Value scalar,
+                                Type referenceType,
+                                ArrayRef<OpFoldResult> referenceShape) {
+  // If the result type is a scalar, return the unmodified scalar constant.
+  auto tensorType = dyn_cast<TensorType>(referenceType);
+  if (!tensorType) {
+    assert(referenceShape.empty());
+    return scalar;
+  }
+
+  // Create tensor splat
+  auto tensorConstant =
+      builder.create<tensor::SplatOp>(loc, scalar, referenceShape);
+  return tensorConstant;
+}
+
+// Reshape an unranked tensor into a 1D ranked tensor.
+//
+// - input
+//   Unranked tensor.
+//
+// Return values:
+//
+// - flatInput
+//   1D ranked, dynamically shaped tensor.
+//
+// - inputShape
+//   1D extent tensor containing the shape of the original unranked input.
+//
+std::pair<Value, Value> flattenUnrankedTensor(OpBuilder &builder, Location loc,
+                                              Value input) {
+  // Get unranked input shape and total size
+  auto *context = builder.getContext();
+  auto shapeType = shape::getExtentTensorType(context);
+  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+  Value inputSize = builder.create<shape::NumElementsOp>(
+      loc, builder.getIndexType(), inputShape);
+
+  // Turn input size into 1D tensor
+  auto flatShapeType = shape::getExtentTensorType(context, 1);
+  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+      loc, flatShapeType, inputSize);
+
+  // Reshape input tensor into 1D
+  auto inputType = cast<UnrankedTensorType>(input.getType());
+  auto elementType = inputType.getElementType();
+  auto flatInputType =
+      RankedTensorType::get({ShapedType::kDynamic}, elementType);
+  auto flatInput = builder.create<tensor::ReshapeOp>(
+      loc, flatInputType, input, flatInputShape);
+  return std::make_pair(flatInput, inputShape);
+}
+
+// Reshape an unranked tensor into a 3D ranked tensor where the central
+// dimension of the result tensor corresponds to dimension 'axis' of the input
+// tensor.
+//
+// - input
+//   Unranked tensor.
+//
+// - axis
+//   Index of the input dimension around which other input dimiensions will be
+//   collapsed.
+//
+// - axisSize
+//   Size of input dimension 'axis'.
+//
+// Return values:
+//
+// - flatInput
+//   3D ranked tensor of shape [?, axisSize, ?].
+//
+// - inputShape
+//   1D extent tensor containing the shape of the original unranked input.
+//
+std::pair<Value, Value> flattenUnrankedTensorAroundAxis(OpBuilder &builder,
+                                                        Location loc,
+                                                        Value input,
+                                                        int64_t axis,
+                                                        int64_t axisSize) {
+  // Get full tensor shape
+  auto *context = builder.getContext();
+  auto indexType = builder.getIndexType();
+  auto shapeType = shape::getExtentTensorType(context);
+  auto inputShape = builder.create<shape::ShapeOfOp>(loc, shapeType, input);
+
+  // Get shape and sizes on left and right of axis
+  auto axisValue = builder.create<arith::ConstantIndexOp>(loc, axis);
+  auto axisNextValue = builder.create<arith::ConstantIndexOp>(loc, axis + 1);
+  auto shapeLeft = builder.create<shape::SplitAtOp>(
+      loc, TypeRange{shapeType, shapeType}, inputShape, axisValue)
+      .getResult(0);
+  auto sizeLeft = builder.create<shape::NumElementsOp>(
+      loc, indexType, shapeLeft);
+  auto shapeRight = builder.create<shape::SplitAtOp>(
+      loc, TypeRange{shapeType, shapeType}, inputShape, axisNextValue)
+      .getResult(1);
+  auto sizeRight = builder.create<shape::NumElementsOp>(
+      loc, indexType, shapeRight);
+
+  // Compute flat input shape as a 3-element 1D tensor
+  auto axisSizeValue = builder.create<arith::ConstantIndexOp>(loc, axisSize);
+  auto flatShapeType = shape::getExtentTensorType(context, 3);
+  auto flatInputShape = builder.create<tensor::FromElementsOp>(
+      loc, flatShapeType, ValueRange{sizeLeft, axisSizeValue, sizeRight});
+
+  // Reshape input to 3D tensor
+  auto inputType = cast<UnrankedTensorType>(input.getType());
+  auto elementType = inputType.getElementType();
+  auto flatInputType = RankedTensorType::get(
+      {ShapedType::kDynamic, axisSize, ShapedType::kDynamic}, elementType);
+  auto flatInput = builder.create<tensor::ReshapeOp>(
+      loc, flatInputType, input, flatInputShape);
+
+  return std::make_pair(flatInput, inputShape);
+}
+
+// Reshape an input tensor into its original unranked shape.
+//
+// - input
+//   Ranked tensor.
+//
+// - inputShape
+//   1D extent tensor.
+//
+Value restoreUnrankedTensorShape(OpBuilder &builder, Location loc, Value input,
+                                 Value inputShape) {
+  auto inputType = cast<RankedTensorType>(input.getType());
+  auto elementType = inputType.getElementType();
+  auto unrankedType = UnrankedTensorType::get(elementType);
+  return builder.create<tensor::ReshapeOp>(loc, unrankedType, input, inputShape);
+}
+
+// Create a tensor constant containing all scales in a per-channel quantized
+// type. Example:
+//
+//   !quant.uniform<i8:f32:1, {2.0:10, 3.0:20}>
+//
+// produces
+//
+//   %cst = arith.constant dense<[2.0, 3.0]> : tensor<2xf32>
+//
+Value materializePerChannelScales(OpBuilder &builder, Location loc,
+                                  UniformQuantizedPerAxisType quantizedType) {
+  auto scales = quantizedType.getScales();
+  auto expressedType = quantizedType.getExpressedType();
+  auto scaleAttrs = llvm::map_to_vector(scales, [&](double scale) -> Attribute {
+    return builder.getFloatAttr(expressedType, scale);
+  });
+  auto tensorType = RankedTensorType::get({(int64_t) scales.size()}, expressedType);
+  auto scalesAttr = DenseElementsAttr::get(tensorType, scaleAttrs);
+  return builder.create<arith::ConstantOp>(loc, tensorType, scalesAttr);
+}
+
+// Create a tensor constant containing all zero points in a per-channel
+// quantized type. Example:
+//
+//   !quant.uniform<i8:f32:1, {2.0:10, 3.0:20}>
+//
+// produces
+//
+//   %cst = arith.constant dense<[10, 20]> : tensor<2xi8>
+//
+Value materializePerChannelZeroPoints(
+    OpBuilder &builder, Location loc,
+    UniformQuantizedPerAxisType quantizedType) {
+  auto zeroPoints = quantizedType.getZeroPoints();
+  auto storageType = quantizedType.getStorageType();
+  auto zeroPointAttrs = llvm::map_to_vector(
+      zeroPoints,
+      [&](int64_t zeroPoint) -> Attribute {
+        return builder.getIntegerAttr(storageType, zeroPoint);
+      });
+  auto tensorType =
+      RankedTensorType::get({(int64_t)zeroPoints.size()}, storageType);
+  auto zeroPointsAttr = DenseElementsAttr::get(tensorType, zeroPointAttrs);
+  return builder.create<arith::ConstantOp>(loc, tensorType, zeroPointsAttr);
+}
+
+// Clamp the given scalar or tensor input using the storage bounds encoded in
+// the given quantized type, if present.
+//
+// - input
+//   Scalar or ranked tensor input. The element type must match the storage type
+//   of 'quantizedType'.
+//
+// - inputShape
+//   If 'input' is a tensor, combination of attributes/values representing its
+//   static/dynamic dimensions. If 'input' is a scalar, empty list.
+//
+// - quantizedType
+//   Per-axis or per-channel quantized type.
+Value clampScalarOrTensor(OpBuilder &builder, Location loc, Value input,
+                          ArrayRef<OpFoldResult> inputShape,
+                          QuantizedType quantizedType) {
+  // If quantized type does not narrow down the storage type range, there is
+  // nothing to do.
+  if (!quantizedType.hasStorageTypeBounds())
+    return input;
+
+  // Materialize bounds
+  auto inputType = input.getType();
+  auto storageType = quantizedType.getStorageType();
+  auto storageMinScalar = builder.create<arith::ConstantIntOp>(
+      loc, quantizedType.getStorageTypeMin(), storageType);
+  auto storageMaxScalar = builder.create<arith::ConstantIntOp>(
+      loc, quantizedType.getStorageTypeMax(), storageType);
+  auto storageMin = getScalarOrTensorConstant(builder, loc, storageMinScalar,
+                                              inputType, inputShape);
+  auto storageMax = getScalarOrTensorConstant(builder, loc, storageMaxScalar,
+                                              inputType, inputShape);
+
+  // Clamp
+  if (quantizedType.isSigned()) {
+    input = builder.create<arith::MaxSIOp>(loc, input, storageMin);
+    input = builder.create<arith::MinSIOp>(loc, input, storageMax);
+  } else {
+    input = builder.create<arith::MaxUIOp>(loc, input, storageMin);
+    input = builder.create<arith::MinUIOp>(loc, input, storageMax);
+  }
+  return input;
+}
+
+// Emit op 'arith.fptosi' or 'arith.fptoui'.
+Value convertFloatToInteger(OpBuilder &builder, Location loc, Value input,
+                            Type resultType, bool isSigned) {
+  if (isSigned)
+    return builder.create<arith::FPToSIOp>(loc, resultType, input);
+  return builder.create<arith::FPToUIOp>(loc, resultType, input);
+}
+
+// Emit op 'arith.sitofp' or 'arith.uitofp'.
+Value convertIntegerToFloat(OpBuilder &builder, Location loc, Value input,
+                            Type resultType, bool isSigned) {
+  if (isSigned)
+    return builder.create<arith::SIToFPOp>(loc, resultType, input);
+  return builder.create<arith::UIToFPOp>(loc, resultType, input);
+}
+
+// Quantize a scalar or ranked tensor value. The stored value is clamped using 
+// the storage bounds encoded in the given quantized type.
+//
+// See function 'convertRanked()' below for a description of the arguments.
+Value quantizeValue(OpBuilder &builder, Location loc, Value input,
+                    ArrayRef<OpFoldResult> inputShape, Value scale,
+                    Value zeroPoint, QuantizedType quantizedType) {
+  // Convert scale to tensor if necessary
+  auto inputType = input.getType();
+  scale = getScalarOrTensorConstant(
+      builder, loc, scale, inputType, inputShape);
+
+  // Scale input
+  auto scaledValue = builder.create<arith::DivFOp>(loc, input, scale);
+
+  // Skip unnecessary computations if no zero point is given
+  Value storedValueFloat = scaledValue;
+  if (!matchPattern(zeroPoint, m_Zero())) {
+    // Convert zero point to tensor if necessary
+    zeroPoint = getScalarOrTensorConstant(builder, loc, zeroPoint, inputType,
+                                          inputShape);
+
+    // Convert zero point from storage to expressed type
+    zeroPoint = convertIntegerToFloat(builder, loc, zeroPoint,
+                                      scale.getType(),
+                                      quantizedType.isSigned());
+
+    // Add zero point to stored value
+    storedValueFloat =
+        builder.create<arith::AddFOp>(loc, scaledValue, zeroPoint);
+  }
+
+  // Convert stored value to storage type
+  auto storageScalarOrTensorType =
+      getScalarOrTensorType(quantizedType.getStorageType(), inputType);
+  auto storedValueInt = convertFloatToInteger(
+      builder, loc, storedValueFloat, storageScalarOrTensorType,
+      quantizedType.isSigned());
+
+  // Clamp stored value it if the storage type is bound
+  auto storedValueClamped = clampScalarOrTensor(builder, loc, storedValueInt,
+                                                inputShape, quantizedType);
+  return storedValueClamped;
+}
+
+// Dequantize a scalar or ranked tensor input.
+//
+// See function 'convertRanked()' below for a description of the arguments.
+Value dequantizeValue(OpBuilder &builder, Location loc, Value input,
+                      ArrayRef<OpFoldResult> inputShape, Value scale,
+                      Value zeroPoint, QuantizedType quantizedType) {
+  // Convert scale to tensor if necessary
+  auto inputType = input.getType();
+  scale = getScalarOrTensorConstant(
+      builder, loc, scale, inputType, inputShape);
+
+  // Convert stored value to float
+  auto result = convertIntegerToFloat(
+      builder, loc, input, scale.getType(), quantizedType.isSigned());
+
+  // Skip unnecessary computations if no zero point is given
+  if (!matchPattern(zeroPoint, m_Zero())) {
+    // Convert zero point to tensor if necessary
+    zeroPoint = getScalarOrTensorConstant(builder, loc, zeroPoint, inputType,
+                                          inputShape);
+
+    // Convert zero point from storage to expressed type
+    zeroPoint = convertIntegerToFloat(builder, loc, zeroPoint,
+                                      scale.getType(),
+                                      quantizedType.isSigned());
+
+    // Subtract zero point to stored value
+    result = builder.create<arith::SubFOp>(loc, result, zeroPoint);
+  }
+
+  // Multiply by scale
+  result = builder.create<arith::MulFOp>(loc, result, scale);
+  return result;
+}
+
+// Convert a scalar or ranked tensor input with the given scale and zero point
+// values.
+//
+// - input
+//   Scalar or ranked tensor value.
+//
+// - inputShape
+//   If 'input' is a tensor, combination or attributes/values representing its
+//   static/dynamic dimensions. If 'input' is a scalar, empty list.
+//
+// - scale
+//   Scale as a floating-point scalar value.
+//
+// - zeroPoint
+//   Zero point as an integer scalar value.
+//
+// - quantizedType
+//   Scalar quantized type of the result ('quant.qcast') or of the input
+//   ('quant.dcast').
+//
+Value convertRanked(OpBuilder &builder, Location loc, Operation *op,
+                    Value input, ArrayRef<OpFoldResult> inputShape, Value scale,
+                    Value zeroPoint, QuantizedType quantizedType) {
+  if (isa<QuantizeCastOp>(op))
+    return quantizeValue(builder, loc, input, inputShape, scale, zeroPoint,
+                         quantizedType);
+  if (isa<DequantizeCastOp>(op))
+    return dequantizeValue(builder, loc, input, inputShape, scale, zeroPoint,
+                           quantizedType);
+  llvm_unreachable("unexpected quant op");
+}
+
+// Convert an operation using per-layer quantization with a scalar or ranked
+// tensor input.
+//
+// - op
+//   'quant.dcast' or 'quant.qcast' op.
+//
+// - input
+//   Scalar or ranked tensor.
+//
+// - quantizedType
+//   Per-layer quantized type.
+//
+Value convertPerLayerRanked(OpBuilder &builder, Location loc, Operation *op,
+                            Value input, UniformQuantizedType quantizedType) {
+  // Create scale and zero point constants
+  auto expressedType = quantizedType.getExpressedType();
+  auto storageType = quantizedType.getStorageType();
+  auto scaleAttr =
+      builder.getFloatAttr(expressedType, quantizedType.getScale());
+  auto scale = builder.create<arith::ConstantOp>(loc, expressedType, scaleAttr);
+  auto zeroPointAttr =
+      builder.getIntegerAttr(storageType, quantizedType.getZeroPoint());
+  auto zeroPoint =
+      builder.create<arith::ConstantOp>(loc, storageType, zeroPointAttr);
+
+  auto inputShape = getScalarOrTensorShape(builder, loc, input);
+  return convertRanked(builder, loc, op, input, inputShape, scale, zeroPoint,
+                       quantizedType);
+}
+
+// Convert an operation using per-layer quantization.
+//
+// - op
+//   'quant.dcast' or 'quant.qcast' op.
+//
+// - input
+//   Scalar, ranked tensor, or unranked tensor.
+//
+// - quantizedType
+//   Per-layer quantized type.
+//
+Value convertPerLayer(OpBuilder &builder, Location loc, Operation *op,
+                      Value input, UniformQuantizedType quantizedType) {
+  // Flatten input if unranked
+  bool isUnranked = isa<UnrankedTensorType>(input.getType());
+  Value inputShape;
+  if (isUnranked)
+    std::tie(input, inputShape) = flattenUnrankedTensor(builder, loc, input);
+
+  // Process ranked tensor
+  auto result = convertPerLayerRanked(builder, loc, op, input, quantizedType);
+
+  // Restore original shape if unranked
+  if (isUnranked)
+    result = restoreUnrankedTensorShape(builder, loc, result, inputShape);
+
+  return result;
+}
+
+// Convert an operation using per-channel quantization and a scalar or ranked
+// tensor as an input.
+//
+// - op
+//   'quant.dcast' or 'quant.qcast' op.
+//
+// - input
+//   Scalar or ranked tensor.
+//
+// - quantizedType
+//   Per-channel quantized type.
+//
+Value convertPerChannelRanked(OpBuilder &builder, Location loc, Operation *op,
+                              Value input,
+                              UniformQuantizedPerAxisType quantizedType,
+                              int64_t channelAxis) {
+  auto *context = builder.getContext();
+
+  auto inputType = cast<RankedTensorType>(input.getType());
+  auto inputRank = inputType.getRank();
+
+  auto scales = materializePerChannelScales(builder, loc, quantizedType);
+  auto zeroPoints =
+      materializePerChannelZeroPoints(builder, loc, quantizedType);
+
+  auto elementType = isa<FloatType>(inputType.getElementType())
+                         ? quantizedType.getStorageType()
+                         : quantizedType.getExpressedType();
+  auto initShape = tensor::getMixedSizes(builder, loc, input);
+  Value init = builder.create<tensor::EmptyOp>(loc, initShape, elementType);
+
+  SmallVector<utils::IteratorType> iteratorTypes(
+      inputRank, utils::IteratorType::parallel);
+  auto channelAxisAffineMap = AffineMap::get(
+      inputRank, 0, builder.getAffineDimExpr(channelAxis), context);
+  SmallVector<AffineMap> indexingMaps{
+    builder.getMultiDimIdentityMap(inputRank),
+    channelAxisAffineMap,
+    channelAxisAffineMap,
+    builder.getMultiDimIdentityMap(inputRank)
+  };
+  auto result = builder.create<linalg::GenericOp>(
+      loc,
+      init.getType(),  // resultType
+      ValueRange{input, scales, zeroPoints},  // inputs
+      ValueRange{init},  // outputs
+      indexingMaps,
+      iteratorTypes,
+      [&](OpBuilder& builder, Location loc, ValueRange args) {
+        assert(args.size() == 4);
+        auto input = args[0];
+        auto scale = args[1];
+        auto zeroPoint = args[2];
+
+        auto result = convertRanked(builder, loc, op, input, {}, scale,
+                                    zeroPoint, quantizedType);
+
+        builder.create<linalg::YieldOp>(loc, result);
+      })
+      .getResult(0);
+
+  return result;
+}
+
+// Convert an operation using per-channel quantization.
+//
+// - op
+//   'quant.dcast' or 'quant.qcast' op.
+//
+// - input
+//   Scalar, ranked tensor, or unranked tensor.
+//
+// - quantizedType
+//   Per-channel quantized type.
+//
+Value convertPerChannel(OpBuilder &builder, Location loc, Operation *op,
+                        Value input,
+                        UniformQuantizedPerAxisType quantizedType) {
+  // Flatten unranked tensor into a 3D ranked tensor if necessary
+  bool isUnranked = isa<UnrankedTensorType>(input.getType());
+  int64_t channelAxis = quantizedType.getQuantizedDimension();
+  int64_t channelAxisSize = (int64_t) quantizedType.getScales().size();
+  Value inputShape;
+  if (isUnranked) {
+    std::tie(input, inputShape) = flattenUnrankedTensorAroundAxis(
+        builder, loc, input, channelAxis, channelAxisSize);
+    channelAxis = 1;
+  }
+
+  // Work on a ranked tensor
+  auto result = convertPerChannelRanked(builder, loc, op, input, quantizedType,
+                                        channelAxis);
+
+  // Restore original tensor shape if unranked
+  if (isUnranked)
+    result = restoreUnrankedTensorShape(builder, loc, result, inputShape);
+
+  return result;
+}
+
+// Convert a quantization operation.
+//
+// - op
+//   'quant.dcast' or 'quant.qcast' op.
+//
+// - input
+//   Scalar, ranked tensor, or unranked tensor. The element type matches
+//   the storage type (quant.dcast) or expressed type (quant.qcast) of
+//   'quantizedType'.
+//
+// - quantizedType
+//   Per-layer or per-channel quantized type.
+//
+Value convertQuantized(OpBuilder &builder, Location loc, Operation *op,
+                       Value input, Type quantizedType) {
+  if (auto uniformQuantizedType = dyn_cast<UniformQuantizedType>(quantizedType))
+    return convertPerLayer(builder, loc, op, input, uniformQuantizedType);
+
+  if (auto uniformQuantizedPerAxisType =
+          dyn_cast<UniformQuantizedPerAxisType>(quantizedType))
+    return convertPerChannel(builder, loc, op, input,
+                             uniformQuantizedPerAxisType);
+
+  llvm_unreachable("unexpected quantized type");
+}
+
+// Lowering pattern for 'quant.dcast'
+struct DequantizeCastOpConversion : public OpConversionPattern<quant::DequantizeCastOp> {
+  using OpConversionPattern<quant::DequantizeCastOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(quant::DequantizeCastOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    auto input = op.getInput();
+    auto quantizedType =
+        cast<QuantizedType>(getScalarType(op.getInput().getType()));
+
+    // Convert quantized input to storage type
+    auto storageScalarOrTensorType =
+        getScalarOrTensorType(quantizedType.getStorageType(), input.getType());
+    input = rewriter.create<quant::StorageCastOp>(
+        loc, storageScalarOrTensorType, input);
+
+    auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
+// Lowering pattern for 'quant.qcast'
+struct QuantizeCastOpConversion : public OpConversionPattern<quant::QuantizeCastOp> {
+  using OpConversionPattern<quant::QuantizeCastOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(quant::QuantizeCastOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto loc = op.getLoc();
+    auto input = op.getInput();
+    auto quantizedType = getScalarType(op.getResult().getType());
+
+    // Flatten unranked tensor input
+    auto result = convertQuantized(rewriter, loc, op, input, quantizedType);
+
+    // Cast stored value to result quantized value
+    rewriter.replaceOpWithNewOp<quant::StorageCastOp>(
+        op, op.getResult().getType(), result);
+    return success();
+  }
+};
+
+struct LowerQuantOps : public impl::LowerQuantOpsBase<LowerQuantOps> {
+  void runOnOperation() override {
+    RewritePatternSet patterns(&getContext());
+    populateLowerQuantOpsPatterns(patterns);
+
+    ConversionTarget target(getContext());
+    target.addLegalOp<quant::StorageCastOp>();
+    target.addIllegalDialect<quant::QuantDialect>();
+    target.addLegalDialect<
+      arith::ArithDialect,
+      linalg::LinalgDialect,
+      shape::ShapeDialect,
+      tensor::TensorDialect
+    >();
+
+    if (failed(applyPartialConversion(getOperation(), target,
+                                      std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+void populateLowerQuantOpsPatterns(RewritePatternSet &patterns) {
+  patterns.add<
+    DequantizeCastOpConversion,
+    QuantizeCastOpConversion
+  >(patterns.getContext());
+}
+
+} // namespace quant
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
new file mode 100644
index 0000000000000..8996eff61a39c
--- /dev/null
+++ b/mlir/lib/Dialect/Quant/Transforms/StripFuncQuantTypes.cpp
@@ -0,0 +1,114 @@
+//===- StripFuncQuantTypes.cpp - Strip quantized types --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Strips quantized types from function headers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/Func/Transforms/FuncConversions.h"
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
+#include "mlir/Dialect/Quant/Transforms/Passes.h"
+#include "mlir/Dialect/Shape/IR/Shape.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/Matchers.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+namespace mlir {
+namespace quant {
+
+#define GEN_PASS_DEF_STRIPFUNCQUANTTYPES
+#include "mlir/Dialect/Quant/Transforms/Passes.h.inc"
+
+namespace {
+
+class QuantizedTypeConverter : public TypeConverter {
+
+  static Type convertQuantizedType(QuantizedType quantizedType) {
+    return quantizedType.getStorageType();
+  }
+  
+  static Type convertTensorType(TensorType tensorType) {
+    if (auto quantizedType = dyn_cast<QuantizedType>(tensorType.getElementType()))
+      return tensorType.clone(convertQuantizedType(quantizedType));
+    return tensorType;
+  }
+
+  static Value materializeConversion(OpBuilder &builder, Type type,
+                                     ValueRange inputs, Location loc) {
+    assert(inputs.size() == 1);
+    return builder.create<quant::StorageCastOp>(loc, type, inputs[0]);
+  }
+
+public:
+
+  explicit QuantizedTypeConverter() {
+    addConversion([](Type type) { return type; });
+    addConversion(convertQuantizedType);
+    addConversion(convertTensorType);
+
+    addArgumentMaterialization(materializeConversion);
+    addSourceMaterialization(materializeConversion);
+    addTargetMaterialization(materializeConversion);
+  }
+};
+
+// Conversion pass
+class StripFuncQuantTypes : public impl::StripFuncQuantTypesBase<StripFuncQuantTypes> {
+
+  // Return whether a type is considered legal when occurring in the header of
+  // a function or as an operand to a 'return' op.
+  static bool isLegalType(Type type) {
+    if (auto tensorType = dyn_cast<TensorType>(type))
+      return isLegalType(tensorType.getElementType());
+    return !isa<quant::QuantizedType>(type);
+  }
+
+public:
+
+  void runOnOperation() override {
+    
+    auto moduleOp = cast<ModuleOp>(getOperation());
+    auto* context = &getContext();
+
+    QuantizedTypeConverter typeConverter;
+    ConversionTarget target(*context);
+    RewritePatternSet patterns(context);
+
+    // Mark func.func, func.return, and func.call illegal if they contain any
+    // quantized types.
+    target.addDynamicallyLegalOp<func::FuncOp>([&](func::FuncOp op) {
+      return typeConverter.isSignatureLegal(op.getFunctionType()) &&
+             typeConverter.isLegal(&op.getBody());
+    });
+    target.addDynamicallyLegalOp<func::ReturnOp>(
+        [&](func::ReturnOp op) { return typeConverter.isLegal(op); });
+    target.addDynamicallyLegalOp<func::CallOp>(
+        [&](func::CallOp op) { return typeConverter.isLegal(op); });
+
+    // Register conversion patterns
+    populateFunctionOpInterfaceTypeConversionPattern<func::FuncOp>(
+        patterns, typeConverter);
+    populateReturnOpTypeConversionPattern(patterns, typeConverter);
+    populateCallOpTypeConversionPattern(patterns, typeConverter);
+
+    // Apply conversion
+    if (failed(applyPartialConversion(moduleOp, target, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+} // namespace quant
+} // namespace mlir
+
diff --git a/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp b/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
index 8c69729824691..fb27640bfd278 100644
--- a/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/FakeQuantSupport.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/FakeQuantSupport.h"
-#include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/Dialect/Quant/IR/QuantTypes.h"
+#include "mlir/Dialect/Quant/Utils/FakeQuantSupport.h"
 
 using namespace mlir;
 using namespace mlir::quant;
diff --git a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
index 408701f80444a..62c7a7128d63a 100644
--- a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/UniformSupport.h"
+#include "mlir/Dialect/Quant/Utils/UniformSupport.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include <numeric>
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 6d47ff3890977..d1c9fd2d217da 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -772,6 +772,70 @@ LoopNest mlir::scf::buildLoopNest(
                        });
 }
 
+SmallVector<Value>
+mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
+                                      OpOperand &operand, Value replacement,
+                                      const ValueTypeCastFnTy &castFn) {
+  assert(operand.getOwner() == forOp);
+  Type oldType = operand.get().getType(), newType = replacement.getType();
+
+  // 1. Create new iter operands, exactly 1 is replaced.
+  assert(operand.getOperandNumber() >= forOp.getNumControlOperands() &&
+         "expected an iter OpOperand");
+  assert(operand.get().getType() != replacement.getType() &&
+         "Expected a different type");
+  SmallVector<Value> newIterOperands;
+  for (OpOperand &opOperand : forOp.getInitArgsMutable()) {
+    if (opOperand.getOperandNumber() == operand.getOperandNumber()) {
+      newIterOperands.push_back(replacement);
+      continue;
+    }
+    newIterOperands.push_back(opOperand.get());
+  }
+
+  // 2. Create the new forOp shell.
+  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
+      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
+      forOp.getStep(), newIterOperands);
+  newForOp->setAttrs(forOp->getAttrs());
+  Block &newBlock = newForOp.getRegion().front();
+  SmallVector<Value, 4> newBlockTransferArgs(newBlock.getArguments().begin(),
+                                             newBlock.getArguments().end());
+
+  // 3. Inject an incoming cast op at the beginning of the block for the bbArg
+  // corresponding to the `replacement` value.
+  OpBuilder::InsertionGuard g(rewriter);
+  rewriter.setInsertionPoint(&newBlock, newBlock.begin());
+  BlockArgument newRegionIterArg = newForOp.getTiedLoopRegionIterArg(
+      &newForOp->getOpOperand(operand.getOperandNumber()));
+  Value castIn = castFn(rewriter, newForOp.getLoc(), oldType, newRegionIterArg);
+  newBlockTransferArgs[newRegionIterArg.getArgNumber()] = castIn;
+
+  // 4. Steal the old block ops, mapping to the newBlockTransferArgs.
+  Block &oldBlock = forOp.getRegion().front();
+  rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
+
+  // 5. Inject an outgoing cast op at the end of the block and yield it instead.
+  auto clonedYieldOp = cast<scf::YieldOp>(newBlock.getTerminator());
+  rewriter.setInsertionPoint(clonedYieldOp);
+  unsigned yieldIdx =
+      newRegionIterArg.getArgNumber() - forOp.getNumInductionVars();
+  Value castOut = castFn(rewriter, newForOp.getLoc(), newType,
+                         clonedYieldOp.getOperand(yieldIdx));
+  SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
+  newYieldOperands[yieldIdx] = castOut;
+  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
+  rewriter.eraseOp(clonedYieldOp);
+
+  // 6. Inject an outgoing cast op after the forOp.
+  rewriter.setInsertionPointAfter(newForOp);
+  SmallVector<Value> newResults = newForOp.getResults();
+  newResults[yieldIdx] =
+      castFn(rewriter, newForOp.getLoc(), oldType, newResults[yieldIdx]);
+
+  return newResults;
+}
+
 namespace {
 // Fold away ForOp iter arguments when:
 // 1) The op yields the iter arguments.
@@ -973,76 +1037,6 @@ struct SimplifyTrivialLoops : public OpRewritePattern<ForOp> {
   }
 };
 
-/// Perform a replacement of one iter OpOperand of an scf.for to the
-/// `replacement` value which is expected to be the source of a tensor.cast.
-/// tensor.cast ops are inserted inside the block to account for the type cast.
-static SmallVector<Value>
-replaceTensorCastForOpIterArg(PatternRewriter &rewriter, OpOperand &operand,
-                              Value replacement) {
-  Type oldType = operand.get().getType(), newType = replacement.getType();
-  assert(llvm::isa<RankedTensorType>(oldType) &&
-         llvm::isa<RankedTensorType>(newType) &&
-         "expected ranked tensor types");
-
-  // 1. Create new iter operands, exactly 1 is replaced.
-  ForOp forOp = cast<ForOp>(operand.getOwner());
-  assert(operand.getOperandNumber() >= forOp.getNumControlOperands() &&
-         "expected an iter OpOperand");
-  assert(operand.get().getType() != replacement.getType() &&
-         "Expected a different type");
-  SmallVector<Value> newIterOperands;
-  for (OpOperand &opOperand : forOp.getInitArgsMutable()) {
-    if (opOperand.getOperandNumber() == operand.getOperandNumber()) {
-      newIterOperands.push_back(replacement);
-      continue;
-    }
-    newIterOperands.push_back(opOperand.get());
-  }
-
-  // 2. Create the new forOp shell.
-  scf::ForOp newForOp = rewriter.create<scf::ForOp>(
-      forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-      forOp.getStep(), newIterOperands);
-  newForOp->setAttrs(forOp->getAttrs());
-  Block &newBlock = newForOp.getRegion().front();
-  SmallVector<Value, 4> newBlockTransferArgs(newBlock.getArguments().begin(),
-                                             newBlock.getArguments().end());
-
-  // 3. Inject an incoming cast op at the beginning of the block for the bbArg
-  // corresponding to the `replacement` value.
-  OpBuilder::InsertionGuard g(rewriter);
-  rewriter.setInsertionPoint(&newBlock, newBlock.begin());
-  BlockArgument newRegionIterArg = newForOp.getTiedLoopRegionIterArg(
-      &newForOp->getOpOperand(operand.getOperandNumber()));
-  Value castIn = rewriter.create<tensor::CastOp>(newForOp.getLoc(), oldType,
-                                                 newRegionIterArg);
-  newBlockTransferArgs[newRegionIterArg.getArgNumber()] = castIn;
-
-  // 4. Steal the old block ops, mapping to the newBlockTransferArgs.
-  Block &oldBlock = forOp.getRegion().front();
-  rewriter.mergeBlocks(&oldBlock, &newBlock, newBlockTransferArgs);
-
-  // 5. Inject an outgoing cast op at the end of the block and yield it instead.
-  auto clonedYieldOp = cast<scf::YieldOp>(newBlock.getTerminator());
-  rewriter.setInsertionPoint(clonedYieldOp);
-  unsigned yieldIdx =
-      newRegionIterArg.getArgNumber() - forOp.getNumInductionVars();
-  Value castOut = rewriter.create<tensor::CastOp>(
-      newForOp.getLoc(), newType, clonedYieldOp.getOperand(yieldIdx));
-  SmallVector<Value> newYieldOperands = clonedYieldOp.getOperands();
-  newYieldOperands[yieldIdx] = castOut;
-  rewriter.create<scf::YieldOp>(newForOp.getLoc(), newYieldOperands);
-  rewriter.eraseOp(clonedYieldOp);
-
-  // 6. Inject an outgoing cast op after the forOp.
-  rewriter.setInsertionPointAfter(newForOp);
-  SmallVector<Value> newResults = newForOp.getResults();
-  newResults[yieldIdx] = rewriter.create<tensor::CastOp>(
-      newForOp.getLoc(), oldType, newResults[yieldIdx]);
-
-  return newResults;
-}
-
 /// Fold scf.for iter_arg/result pairs that go through incoming/ougoing
 /// a tensor.cast op pair so as to pull the tensor.cast inside the scf.for:
 ///
@@ -1090,9 +1084,13 @@ struct ForOpTensorCastFolder : public OpRewritePattern<ForOp> {
         continue;
 
       // Create a new ForOp with that iter operand replaced.
+      ValueTypeCastFnTy castFn = [](OpBuilder &b, Location loc, Type type,
+                                    Value source) {
+        return b.create<tensor::CastOp>(loc, type, source);
+      };
       rewriter.replaceOp(
-          op, replaceTensorCastForOpIterArg(rewriter, iterOpOperand,
-                                            incomingCast.getSource()));
+          op, replaceAndCastForOpIterArg(rewriter, op, iterOpOperand,
+                                         incomingCast.getSource(), castFn));
       return success();
     }
     return failure();
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 3d6da066875f9..83c9cf69ba036 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -648,15 +648,22 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
   // bounds_range = ub - lb
   // total_iterations = (bounds_range + step - 1) / step
   Type t = lb.getType();
-  Value minus1 =
-      rewriter.create<arith::ConstantOp>(loc, rewriter.getIntegerAttr(t, -1));
-  Value boundsRange = rewriter.create<arith::SubIOp>(loc, ub, lb);
-  Value rangeIncr = rewriter.create<arith::AddIOp>(loc, boundsRange, step);
-  Value rangeDecr = rewriter.create<arith::AddIOp>(loc, rangeIncr, minus1);
-  Value totalIterations = rewriter.create<arith::DivUIOp>(loc, rangeDecr, step);
-
   Value zero =
       rewriter.create<arith::ConstantOp>(loc, rewriter.getIntegerAttr(t, 0));
+  Value one =
+      rewriter.create<arith::ConstantOp>(loc, rewriter.getIntegerAttr(t, 1));
+  Value minusOne =
+      rewriter.create<arith::ConstantOp>(loc, rewriter.getIntegerAttr(t, -1));
+  Value stepLessZero = rewriter.create<arith::CmpIOp>(
+      loc, arith::CmpIPredicate::slt, step, zero);
+  Value stepDecr =
+      rewriter.create<arith::SelectOp>(loc, stepLessZero, one, minusOne);
+
+  Value rangeDiff = rewriter.create<arith::SubIOp>(loc, ub, lb);
+  Value rangeIncrStep = rewriter.create<arith::AddIOp>(loc, rangeDiff, step);
+  Value rangeDecr =
+      rewriter.create<arith::AddIOp>(loc, rangeIncrStep, stepDecr);
+  Value totalIterations = rewriter.create<arith::DivSIOp>(loc, rangeDecr, step);
 
   SmallVector<Value> predicates(maxStage + 1);
   for (int64_t i = 0; i < maxStage; i++) {
@@ -665,7 +672,7 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter,
     Value minusI =
         rewriter.create<arith::ConstantOp>(loc, rewriter.getIntegerAttr(t, -i));
     Value iterI = rewriter.create<arith::AddIOp>(
-        loc, rewriter.create<arith::AddIOp>(loc, totalIterations, minus1),
+        loc, rewriter.create<arith::AddIOp>(loc, totalIterations, minusOne),
         minusI);
     // newLastIter = lb + step * iterI
     Value newlastIter = rewriter.create<arith::AddIOp>(
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index 03876a7c64d07..dbd573f96a79f 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Dialect/Quant/QuantOps.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
@@ -380,7 +380,7 @@ struct ConcatSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
 
   LogicalResult matchAndRewrite(tosa::SliceOp sliceOp,
                                 PatternRewriter &rewriter) const override {
-    Value sliceInput = sliceOp.getInput();
+    Value sliceInput = sliceOp.getInput1();
     auto concatOp = sliceInput.getDefiningOp<tosa::ConcatOp>();
     if (!concatOp)
       return rewriter.notifyMatchFailure(
@@ -919,11 +919,11 @@ OpFoldResult ResizeOp::fold(FoldAdaptor adaptor) {
 }
 
 OpFoldResult ReverseOp::fold(FoldAdaptor adaptor) {
-  auto operand = getInput();
+  auto operand = getInput1();
   auto operandTy = llvm::cast<ShapedType>(operand.getType());
   auto axis = getAxis();
   auto operandAttr =
-      llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getInput());
+      llvm::dyn_cast_if_present<SplatElementsAttr>(adaptor.getInput1());
   if (operandAttr)
     return operandAttr;
 
@@ -936,16 +936,16 @@ OpFoldResult ReverseOp::fold(FoldAdaptor adaptor) {
 }
 
 OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
-  auto inputTy = llvm::dyn_cast<RankedTensorType>(getInput().getType());
+  auto inputTy = llvm::dyn_cast<RankedTensorType>(getInput1().getType());
   auto outputTy = llvm::dyn_cast<RankedTensorType>(getType());
 
   if (!inputTy || !outputTy)
     return {};
 
   if (inputTy == outputTy && inputTy.hasStaticShape())
-    return getInput();
+    return getInput1();
 
-  if (!adaptor.getInput())
+  if (!adaptor.getInput1())
     return {};
 
   // Cannot create an ElementsAttr from non-int/float/index types
@@ -953,7 +953,7 @@ OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
       !outputTy.getElementType().isIntOrIndexOrFloat())
     return {};
 
-  auto operand = llvm::cast<ElementsAttr>(adaptor.getInput());
+  auto operand = llvm::cast<ElementsAttr>(adaptor.getInput1());
   if (operand.isSplat() && outputTy.hasStaticShape()) {
     return SplatElementsAttr::get(outputTy, operand.getSplatValue<Attribute>());
   }
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index 6dce3d03066c9..1f3e19fe90c6d 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -14,7 +14,7 @@
 
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Mesh/Interfaces/ShardingInterface.h"
-#include "mlir/Dialect/Quant/QuantOps.h"
+#include "mlir/Dialect/Quant/IR/Quant.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
 #include "mlir/Dialect/Tosa/Utils/ShapeUtils.h"
@@ -850,7 +850,7 @@ LogicalResult tosa::SliceOp::inferReturnTypeComponents(
 }
 
 LogicalResult tosa::SliceOp::verify() {
-  auto inputType = llvm::dyn_cast<RankedTensorType>(getInput().getType());
+  auto inputType = llvm::dyn_cast<RankedTensorType>(getInput1().getType());
   if (!inputType)
     return success();
 
@@ -869,7 +869,7 @@ LogicalResult tosa::TableOp::inferReturnTypeComponents(
     MLIRContext *context, ::std::optional<Location> location,
     TableOp::Adaptor adaptor,
     SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
-  ShapeAdaptor inputShape(adaptor.getInput().getType());
+  ShapeAdaptor inputShape(adaptor.getInput1().getType());
 
   if (!inputShape.hasRank()) {
     inferredReturnShapes.push_back(ShapedTypeComponents());
@@ -882,7 +882,7 @@ LogicalResult tosa::TableOp::inferReturnTypeComponents(
 }
 
 LogicalResult tosa::TableOp::verify() {
-  TensorType inputType = getInput().getType();
+  TensorType inputType = getInput1().getType();
   TensorType outputType = getOutput().getType();
 
   if (inputType.hasRank() && outputType.hasRank() &&
@@ -1973,7 +1973,7 @@ void IfOp::print(OpAsmPrinter &p) {
 }
 
 LogicalResult ReverseOp::verify() {
-  TensorType inputType = getInput().getType();
+  TensorType inputType = getInput1().getType();
   TensorType outputType = getOutput().getType();
   int32_t reverseAxis = getAxis();
 
diff --git a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
index b01d3183af135..547d120404aba 100644
--- a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
+++ b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
@@ -124,7 +124,7 @@ getConstantIntValues(ArrayRef<OpFoldResult> ofrs) {
     auto cv = getConstantIntValue(ofr);
     if (!cv.has_value())
       failed = true;
-    return cv.has_value() ? cv.value() : 0;
+    return cv.value_or(0);
   });
   if (failed)
     return std::nullopt;
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index ad4e42b31962e..8fcef54f12edf 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -1796,6 +1796,62 @@ struct DropUnitDimsFromTransposeOp final
   }
 };
 
+/// A pattern to drop unit dims from the iter_args of an scf.for.
+///
+/// Example:
+///
+///  BEFORE:
+///  ```mlir
+///  %res = scf.for ... iter_args(%iter = %init) -> vector<[4]x1x1x4xf32> {
+///    ...
+///    scf.yield %
+///  }
+///  ```
+///
+///  AFTER:
+///  ```mlir
+///  %drop = vector.shape_cast %init
+///    : vector<4x1x1x[4]xf32> to vector<4x[4]xf32>
+///  %new_loop = scf.for ... iter_args(%iter = %drop) -> vector<[4]x4xf32> {
+///    %new_iter = vector.shape_cast %iter
+///      : vector<[4]x4xf32> to vector<[4]x1x1x4xf32>
+///    ...
+///  }
+///  %res = vector.shape_cast %new_loop
+///    : vector<[4]x4xf32> to vector<[4]x1x1x4xf32>
+///  ```
+struct DropUnitDimsFromScfForOp final : OpRewritePattern<scf::ForOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(scf::ForOp forOp,
+                                PatternRewriter &rewriter) const override {
+    /// Find the first iter_arg with droppable unit dims. Further applications
+    /// of this pattern will apply to later arguments.
+    for (OpOperand &operand : forOp.getInitArgsMutable()) {
+      auto vectorType = dyn_cast<VectorType>(operand.get().getType());
+      if (!vectorType)
+        continue;
+
+      VectorType newVectorType = dropNonScalableUnitDimFromType(vectorType);
+      if (vectorType == newVectorType)
+        continue;
+
+      // Create a new ForOp with that iter operand replaced.
+      auto castFn = [](OpBuilder &b, Location loc, Type type, Value source) {
+        return b.create<vector::ShapeCastOp>(loc, type, source);
+      };
+
+      Value replacement =
+          castFn(rewriter, forOp.getLoc(), newVectorType, operand.get());
+      rewriter.replaceOp(forOp,
+                         replaceAndCastForOpIterArg(rewriter, forOp, operand,
+                                                    replacement, castFn));
+      return success();
+    }
+    return failure();
+  }
+};
+
 /// Pattern to eliminate redundant zero-constants added to reduction operands.
 /// It's enough for there to be one initial zero value, so we can eliminate the
 /// extra ones that feed into `vector.reduction <add>`. These get created by the
@@ -2001,7 +2057,8 @@ void mlir::vector::populateShapeCastFoldingPatterns(RewritePatternSet &patterns,
 void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<DropUnitDimFromElementwiseOps, DropUnitDimsFromTransposeOp,
-               ShapeCastOpFolder>(patterns.getContext(), benefit);
+               ShapeCastOpFolder, DropUnitDimsFromScfForOp>(
+      patterns.getContext(), benefit);
 }
 
 void mlir::vector::populateBubbleVectorBitCastOpPatterns(
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index 5cbd0b090492b..ea3c0723b0775 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -592,6 +592,29 @@ SmallVector<int64_t, 4> AffineMap::compose(ArrayRef<int64_t> values) const {
   return res;
 }
 
+size_t AffineMap::getNumOfZeroResults() const {
+  size_t res = 0;
+  for (auto expr : getResults()) {
+    auto constExpr = dyn_cast<AffineConstantExpr>(expr);
+    if (constExpr && constExpr.getValue() == 0)
+      res++;
+  }
+
+  return res;
+}
+
+AffineMap AffineMap::dropZeroResults() {
+  auto exprs = llvm::to_vector(getResults());
+  SmallVector<AffineExpr> newExprs;
+
+  for (auto expr : getResults()) {
+    auto constExpr = dyn_cast<AffineConstantExpr>(expr);
+    if (!constExpr || constExpr.getValue() != 0)
+      newExprs.push_back(expr);
+  }
+  return AffineMap::get(getNumDims(), getNumSymbols(), newExprs, getContext());
+}
+
 bool AffineMap::isProjectedPermutation(bool allowZeroInResults) const {
   if (getNumSymbols() > 0)
     return false;
diff --git a/mlir/lib/Rewrite/CMakeLists.txt b/mlir/lib/Rewrite/CMakeLists.txt
index a6c39406aa4b3..15b3739e4c633 100644
--- a/mlir/lib/Rewrite/CMakeLists.txt
+++ b/mlir/lib/Rewrite/CMakeLists.txt
@@ -9,6 +9,7 @@ add_mlir_library(MLIRRewrite
 
   DEPENDS
   mlir-generic-headers
+  MLIRConversionPassIncGen
 
   LINK_LIBS PUBLIC
   MLIRIR
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 806c405ac17df..69602af8563aa 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include <cstdlib>
 
@@ -401,6 +402,26 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) {
                                 /*MemoryLimit=*/0,
                                 /*ErrMsg=*/&message))
     return emitLogError("`ptxas`");
+#define DEBUG_TYPE "dump-sass"
+  LLVM_DEBUG({
+    std::optional<std::string> nvdisasm = findTool("nvdisasm");
+    SmallVector<StringRef> nvdisasmArgs(
+        {StringRef("nvdisasm"), StringRef(cubinFile.first)});
+    if (llvm::sys::ExecuteAndWait(nvdisasm.value(), nvdisasmArgs,
+                                  /*Env=*/std::nullopt,
+                                  /*Redirects=*/redirects,
+                                  /*SecondsToWait=*/0,
+                                  /*MemoryLimit=*/0,
+                                  /*ErrMsg=*/&message))
+      return emitLogError("`nvdisasm`");
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> logBuffer =
+        llvm::MemoryBuffer::getFile(logFile->first);
+    if (logBuffer && !(*logBuffer)->getBuffer().empty()) {
+      llvm::dbgs() << "Output:\n" << (*logBuffer)->getBuffer() << "\n";
+      llvm::dbgs().flush();
+    }
+  });
+#undef DEBUG_TYPE
 
   // Invoke `fatbin`.
   message.clear();
diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
index d948ff5eaf176..72d85d796dd4a 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp
@@ -102,6 +102,28 @@ getOverloadedDeclaration(CallIntrinsicOp op, llvm::Intrinsic::ID id,
   return llvm::Intrinsic::getDeclaration(module, id, overloadedArgTysRef);
 }
 
+static llvm::OperandBundleDef
+convertOperandBundle(OperandRange bundleOperands, StringRef bundleTag,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  std::vector<llvm::Value *> operands;
+  operands.reserve(bundleOperands.size());
+  for (Value bundleArg : bundleOperands)
+    operands.push_back(moduleTranslation.lookupValue(bundleArg));
+  return llvm::OperandBundleDef(bundleTag.str(), std::move(operands));
+}
+
+static SmallVector<llvm::OperandBundleDef>
+convertOperandBundles(OperandRangeRange bundleOperands,
+                      ArrayRef<std::string> bundleTags,
+                      LLVM::ModuleTranslation &moduleTranslation) {
+  SmallVector<llvm::OperandBundleDef> bundles;
+  bundles.reserve(bundleOperands.size());
+
+  for (auto [operands, tag] : llvm::zip_equal(bundleOperands, bundleTags))
+    bundles.push_back(convertOperandBundle(operands, tag, moduleTranslation));
+  return bundles;
+}
+
 /// Builder for LLVM_CallIntrinsicOp
 static LogicalResult
 convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
@@ -138,15 +160,15 @@ convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
   // Check the argument types of the call. If the function is variadic, check
   // the subrange of required arguments.
   if (!fn->getFunctionType()->isVarArg() &&
-      op.getNumOperands() != fn->arg_size()) {
+      op.getArgs().size() != fn->arg_size()) {
     return mlir::emitError(op.getLoc(), "intrinsic call has ")
-           << op.getNumOperands() << " operands but " << op.getIntrinAttr()
+           << op.getArgs().size() << " operands but " << op.getIntrinAttr()
            << " expects " << fn->arg_size();
   }
   if (fn->getFunctionType()->isVarArg() &&
-      op.getNumOperands() < fn->arg_size()) {
+      op.getArgs().size() < fn->arg_size()) {
     return mlir::emitError(op.getLoc(), "intrinsic call has ")
-           << op.getNumOperands() << " operands but variadic "
+           << op.getArgs().size() << " operands but variadic "
            << op.getIntrinAttr() << " expects at least " << fn->arg_size();
   }
   // Check the arguments up to the number the function requires.
@@ -164,8 +186,10 @@ convertCallLLVMIntrinsicOp(CallIntrinsicOp op, llvm::IRBuilderBase &builder,
   FastmathFlagsInterface itf = op;
   builder.setFastMathFlags(getFastmathFlags(itf));
 
-  auto *inst =
-      builder.CreateCall(fn, moduleTranslation.lookupValues(op.getOperands()));
+  auto *inst = builder.CreateCall(
+      fn, moduleTranslation.lookupValues(op.getArgs()),
+      convertOperandBundles(op.getOpBundleOperands(), op.getOpBundleTags(),
+                            moduleTranslation));
   if (op.getNumResults() == 1)
     moduleTranslation.mapValue(op->getResults().front()) = inst;
   return success();
@@ -205,17 +229,21 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
   // itself.  Otherwise, this is an indirect call and the callee is the first
   // operand, look it up as a normal value.
   if (auto callOp = dyn_cast<LLVM::CallOp>(opInst)) {
-    auto operands = moduleTranslation.lookupValues(callOp.getOperands());
+    auto operands = moduleTranslation.lookupValues(callOp.getCalleeOperands());
+    SmallVector<llvm::OperandBundleDef> opBundles =
+        convertOperandBundles(callOp.getOpBundleOperands(),
+                              callOp.getOpBundleTags(), moduleTranslation);
     ArrayRef<llvm::Value *> operandsRef(operands);
     llvm::CallInst *call;
     if (auto attr = callOp.getCalleeAttr()) {
-      call = builder.CreateCall(
-          moduleTranslation.lookupFunction(attr.getValue()), operandsRef);
+      call =
+          builder.CreateCall(moduleTranslation.lookupFunction(attr.getValue()),
+                             operandsRef, opBundles);
     } else {
       llvm::FunctionType *calleeType = llvm::cast<llvm::FunctionType>(
           moduleTranslation.convertType(callOp.getCalleeFunctionType()));
       call = builder.CreateCall(calleeType, operandsRef.front(),
-                                operandsRef.drop_front());
+                                operandsRef.drop_front(), opBundles);
     }
     call->setCallingConv(convertCConvToLLVM(callOp.getCConv()));
     call->setTailCallKind(convertTailCallKindToLLVM(callOp.getTailCallKind()));
@@ -312,13 +340,17 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
 
   if (auto invOp = dyn_cast<LLVM::InvokeOp>(opInst)) {
     auto operands = moduleTranslation.lookupValues(invOp.getCalleeOperands());
+    SmallVector<llvm::OperandBundleDef> opBundles =
+        convertOperandBundles(invOp.getOpBundleOperands(),
+                              invOp.getOpBundleTags(), moduleTranslation);
     ArrayRef<llvm::Value *> operandsRef(operands);
     llvm::InvokeInst *result;
     if (auto attr = opInst.getAttrOfType<FlatSymbolRefAttr>("callee")) {
       result = builder.CreateInvoke(
           moduleTranslation.lookupFunction(attr.getValue()),
           moduleTranslation.lookupBlock(invOp.getSuccessor(0)),
-          moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef);
+          moduleTranslation.lookupBlock(invOp.getSuccessor(1)), operandsRef,
+          opBundles);
     } else {
       llvm::FunctionType *calleeType = llvm::cast<llvm::FunctionType>(
           moduleTranslation.convertType(invOp.getCalleeFunctionType()));
@@ -326,7 +358,7 @@ convertOperationImpl(Operation &opInst, llvm::IRBuilderBase &builder,
           calleeType, operandsRef.front(),
           moduleTranslation.lookupBlock(invOp.getSuccessor(0)),
           moduleTranslation.lookupBlock(invOp.getSuccessor(1)),
-          operandsRef.drop_front());
+          operandsRef.drop_front(), opBundles);
     }
     result->setCallingConv(convertCConvToLLVM(invOp.getCConv()));
     moduleTranslation.mapBranch(invOp, result);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 0cba8d80681f1..d788fe1f6165e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3359,8 +3359,7 @@ createDeviceArgumentAccessor(MapInfoData &mapData, llvm::Argument &arg,
   llvm::Value *v = builder.CreateAlloca(arg.getType(), allocaAS);
 
   if (allocaAS != defaultAS && arg.getType()->isPointerTy())
-    v = builder.CreatePointerBitCastOrAddrSpaceCast(
-        v, arg.getType()->getPointerTo(defaultAS));
+    v = builder.CreateAddrSpaceCast(v, builder.getPtrTy(defaultAS));
 
   builder.CreateStore(&arg, v);
 
diff --git a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
index 860bb60726352..910105ddf6958 100644
--- a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
+++ b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
@@ -563,3 +563,36 @@ gpu.module @kernels {
     gpu.return
   }
 }
+
+// -----
+
+// Lowering of subgroup query operations
+
+// CHECK-DAG: llvm.func spir_funccc @_Z18get_sub_group_size() -> i32 attributes {no_unwind, will_return}
+// CHECK-DAG: llvm.func spir_funccc @_Z18get_num_sub_groups() -> i32 attributes {no_unwind, will_return}
+// CHECK-DAG: llvm.func spir_funccc @_Z22get_sub_group_local_id() -> i32 attributes {no_unwind, will_return}
+// CHECK-DAG: llvm.func spir_funccc @_Z16get_sub_group_id() -> i32 attributes {no_unwind, will_return}
+
+
+gpu.module @subgroup_operations {
+// CHECK-LABEL: @gpu_subgroup
+  func.func @gpu_subgroup() {
+    // CHECK:       %[[SG_ID:.*]] = llvm.call spir_funccc @_Z16get_sub_group_id() {no_unwind, will_return} : () -> i32
+    // CHECK-32-NOT:                llvm.zext
+    // CHECK-64           %{{.*}} = llvm.zext %[[SG_ID]] : i32 to i64
+    %0 = gpu.subgroup_id : index
+    // CHECK: %[[SG_LOCAL_ID:.*]] = llvm.call spir_funccc @_Z22get_sub_group_local_id() {no_unwind, will_return}  : () -> i32
+    // CHECK-32-NOT:                llvm.zext
+    // CHECK-64:          %{{.*}} = llvm.zext %[[SG_LOCAL_ID]] : i32 to i64
+    %1 = gpu.lane_id
+    // CHECK:     %[[NUM_SGS:.*]] = llvm.call spir_funccc @_Z18get_num_sub_groups() {no_unwind, will_return} : () -> i32
+    // CHECK-32-NOT:                llvm.zext
+    // CHECK-64:          %{{.*}} = llvm.zext %[[NUM_SGS]] : i32 to i64
+    %2 = gpu.num_subgroups : index
+    // CHECK:     %[[SG_SIZE:.*]] = llvm.call spir_funccc @_Z18get_sub_group_size() {no_unwind, will_return} : () -> i32
+    // CHECK-32-NOT:                llvm.zext
+    // CHECK-64:          %{{.*}} = llvm.zext %[[SG_SIZE]] : i32 to i64
+    %3 = gpu.subgroup_size : index
+    return
+  }
+}
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index 6670e4b186c39..9388d7ef24936 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -218,7 +218,7 @@ func.func @store_unaligned_atomic(%val : f32, %ptr : !llvm.ptr) {
 
 func.func @invalid_call() {
   // expected-error@+1 {{'llvm.call' op must have either a `callee` attribute or at least an operand}}
-  "llvm.call"() : () -> ()
+  "llvm.call"() {op_bundle_sizes = array<i32>} : () -> ()
   llvm.return
 }
 
@@ -286,7 +286,7 @@ func.func @call_non_llvm() {
 
 func.func @call_non_llvm_arg(%arg0 : tensor<*xi32>) {
   // expected-error@+1 {{'llvm.call' op operand #0 must be variadic of LLVM dialect-compatible type}}
-  "llvm.call"(%arg0) : (tensor<*xi32>) -> ()
+  "llvm.call"(%arg0) {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (tensor<*xi32>) -> ()
   llvm.return
 }
 
@@ -1588,7 +1588,7 @@ llvm.func @variadic(...)
 
 llvm.func @invalid_variadic_call(%arg: i32)  {
   // expected-error@+1 {{missing var_callee_type attribute for vararg call}}
-  "llvm.call"(%arg) <{callee = @variadic}> : (i32) -> ()
+  "llvm.call"(%arg) <{callee = @variadic}> {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (i32) -> ()
   llvm.return
 }
 
@@ -1598,7 +1598,7 @@ llvm.func @variadic(...)
 
 llvm.func @invalid_variadic_call(%arg: i32)  {
   // expected-error@+1 {{missing var_callee_type attribute for vararg call}}
-  "llvm.call"(%arg) <{callee = @variadic}> : (i32) -> ()
+  "llvm.call"(%arg) <{callee = @variadic}> {operandSegmentSizes = array<i32: 1, 0>, op_bundle_sizes = array<i32>} : (i32) -> ()
   llvm.return
 }
 
@@ -1655,3 +1655,28 @@ llvm.func @alwaysinline_noinline() attributes { always_inline, no_inline } {
 llvm.func @optnone_requires_noinline() attributes { optimize_none } {
   llvm.return
 }
+
+// -----
+
+llvm.func @foo()
+llvm.func @wrong_number_of_bundle_types() {
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  // expected-error@+1 {{expected 1 types for operand bundle operands for operand bundle #0, but actually got 2}}
+  llvm.call @foo() ["tag"(%0 : i32, i32)] : () -> ()
+  llvm.return
+}
+
+// -----
+
+llvm.func @foo()
+llvm.func @wrong_number_of_bundle_tags() {
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  %1 = llvm.mlir.constant(1 : i32) : i32
+  // expected-error@+1 {{expected 2 operand bundle tags, but actually got 1}}
+  "llvm.call"(%0, %1) <{ op_bundle_tags = ["tag"] }> {
+    callee = @foo,
+    operandSegmentSizes = array<i32: 0, 2>,
+    op_bundle_sizes = array<i32: 1, 1>
+  } : (i32, i32) -> ()
+  llvm.return
+}
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 89d303fcac8ff..62f1de2b7fe7d 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -751,3 +751,86 @@ llvm.func @vector_predication_intrinsics(%A: vector<8xi32>, %B: vector<8xi32>,
          (vector<8xi32>, vector<8xi32>, vector<8xi1>, i32) -> vector<8xi32>
   llvm.return
 }
+
+llvm.func @op_bundle_target()
+
+// CHECK-LABEL: @test_call_with_empty_opbundle
+llvm.func @test_call_with_empty_opbundle() {
+  // CHECK: llvm.call @op_bundle_target() : () -> ()
+  llvm.call @op_bundle_target() [] : () -> ()
+  llvm.return
+}
+
+// CHECK-LABEL: @test_call_with_empty_opbundle_operands
+llvm.func @test_call_with_empty_opbundle_operands() {
+  // CHECK: llvm.call @op_bundle_target() ["tag"()] : () -> ()
+  llvm.call @op_bundle_target() ["tag"()] : () -> ()
+  llvm.return
+}
+
+// CHECK-LABEL: @test_call_with_opbundle
+llvm.func @test_call_with_opbundle() {
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  %1 = llvm.mlir.constant(1 : i32) : i32
+  %2 = llvm.mlir.constant(2 : i32) : i32
+  // CHECK: llvm.call @op_bundle_target() ["tag1"(%{{.+}}, %{{.+}} : i32, i32), "tag2"(%{{.+}} : i32)] : () -> ()
+  llvm.call @op_bundle_target() ["tag1"(%0, %1 : i32, i32), "tag2"(%2 : i32)] : () -> ()
+  llvm.return
+}
+
+// CHECK-LABEL: @test_invoke_with_empty_opbundle
+llvm.func @test_invoke_with_empty_opbundle() attributes { personality = @__gxx_personality_v0 } {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.mlir.constant(3 : i32) : i32
+  // CHECK: llvm.invoke @op_bundle_target() to ^{{.+}} unwind ^{{.+}} : () -> ()
+  llvm.invoke @op_bundle_target() to ^bb2 unwind ^bb1 [] : () -> ()
+
+^bb1:
+  %3 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return
+
+^bb2:
+  llvm.return
+}
+
+// CHECK-LABEL: @test_invoke_with_empty_opbundle_operands
+llvm.func @test_invoke_with_empty_opbundle_operands() attributes { personality = @__gxx_personality_v0 } {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.mlir.constant(3 : i32) : i32
+  // CHECK: llvm.invoke @op_bundle_target() to ^{{.+}} unwind ^{{.+}} ["tag"()] : () -> ()
+  llvm.invoke @op_bundle_target() to ^bb2 unwind ^bb1 ["tag"()] : () -> ()
+
+^bb1:
+  %3 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return
+
+^bb2:
+  llvm.return
+}
+
+// CHECK-LABEL: @test_invoke_with_opbundle
+llvm.func @test_invoke_with_opbundle() attributes { personality = @__gxx_personality_v0 } {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.mlir.constant(3 : i32) : i32
+  // CHECK: llvm.invoke @op_bundle_target() to ^{{.+}} unwind ^{{.+}} ["tag1"(%{{.+}}, %{{.+}} : i32, i32), "tag2"(%{{.+}} : i32)] : () -> ()
+  llvm.invoke @op_bundle_target() to ^bb2 unwind ^bb1 ["tag1"(%0, %1 : i32, i32), "tag2"(%2 : i32)] : () -> ()
+
+^bb1:
+  %3 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return
+
+^bb2:
+  llvm.return
+}
+
+// CHECK-LABEL: @test_call_intrin_with_opbundle
+llvm.func @test_call_intrin_with_opbundle(%arg0 : !llvm.ptr) {
+  %0 = llvm.mlir.constant(1 : i1) : i1
+  %1 = llvm.mlir.constant(16 : i32) : i32
+  // CHECK: llvm.call_intrinsic "llvm.assume"(%{{.+}}) ["align"(%{{.+}}, %{{.+}} : !llvm.ptr, i32)] : (i1) -> ()
+  llvm.call_intrinsic "llvm.assume"(%0) ["align"(%arg0, %1 : !llvm.ptr, i32)] : (i1) -> ()
+  llvm.return
+}
diff --git a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
index 4bfed475d44f6..bfe7a07cb38a5 100644
--- a/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
+++ b/mlir/test/Dialect/Linalg/match-ops-interpreter.mlir
@@ -996,6 +996,21 @@ module attributes { transform.target_tag = "start_here" } {
     } -> tensor<40x10x50x15xf32>
     return %result : tensor<40x10x50x15xf32>
   }
+
+  func.func @generic_min(%arg0: tensor<1x7x4xf32>, %arg1: tensor<4xf32>, %arg2: tensor<1x1x4xf32>) {
+    linalg.generic {
+      indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1 * 2 + d3 * 2, d2)>, 
+      affine_map<(d0, d1, d2, d3) -> (d3)>, 
+      affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>], 
+      iterator_types = ["parallel", "parallel", "parallel", "reduction"]} 
+      ins(%arg0, %arg1 : tensor<1x7x4xf32>, tensor<4xf32>) 
+      outs(%arg2 : tensor<1x1x4xf32>) {
+    ^bb0(%in: f32, %in_1: f32, %out: f32):
+      %5 = arith.minimumf %out, %in : f32
+      linalg.yield %5 : f32
+    } -> tensor<1x1x4xf32>
+    return
+  }
 }
 
 // -----
diff --git a/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir b/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
index ae63ed5f1a410..a6943cf338d42 100644
--- a/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-hoist-pad-build-packing-loop-nest.mlir
@@ -115,8 +115,8 @@ func.func @pad_and_hoist_lhs_transpose(
   //     BUILD-PACKING-LOOP-NEST: %[[PACKED:.*]] = scf.for %{{.*}} -> (tensor<?x12x5xf32>) {
   //     BUILD-PACKING-LOOP-NEST:   tensor.pad %{{.*}}
   //     BUILD-PACKING-LOOP-NEST:     : tensor<?x12xf32> to tensor<5x12xf32>
-  //     BUILD-PACKING-LOOP-NEST:   linalg.generic
-  //     BUILD-PACKING-LOOP-NEST:     -> tensor<12x5xf32>
+  //     BUILD-PACKING-LOOP-NEST:   linalg.transpose
+  //     BUILD-PACKING-LOOP-NEST:     ins({{.*}} : tensor<5x12xf32>) outs({{.*}} : tensor<12x5xf32>)
   //     BUILD-PACKING-LOOP-NEST:   tensor.insert_slice %{{.*}} into %{{.*}}[%{{.*}}, 0, 0] [1, 12, 5] [1, 1, 1]
   // BUILD-PACKING-LOOP-NEST-SAME:   : tensor<12x5xf32> into tensor<?x12x5xf32>
   //     BUILD-PACKING-LOOP-NEST: scf.for %{{.*}} -> (tensor<24x25xf32>)
diff --git a/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir b/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
index 499d9904c06b9..e075ff57666b0 100644
--- a/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-hoist-pad.mlir
@@ -123,17 +123,17 @@ func.func @pad_and_hoist_lhs_transpose(
      -> tensor<24x25xf32>
 {
   //     CHECK: %[[PACKED:.*]] = scf.for %{{.*}} -> (tensor<5x12x5xf32>) {
-  //     CHECK:   tensor.pad %{{.*}}
+  //     CHECK:   %[[PAD:.*]] = tensor.pad %{{.*}}
   //     CHECK:     : tensor<?x12xf32> to tensor<5x12xf32>
-  //     CHECK:   linalg.generic
-  //     CHECK:     -> tensor<12x5xf32>
+  //     CHECK:   linalg.transpose
+  //     CHECK:      ins(%[[PAD]] : tensor<5x12xf32>) outs(%{{.*}} : tensor<12x5xf32>)
   //     CHECK:   tensor.insert_slice %{{.*}} into %{{.*}}[%{{.*}}, 0, 0] [1, 12, 5] [1, 1, 1]
   // CHECK-SAME:   : tensor<12x5xf32> into tensor<5x12x5xf32>
   //     CHECK: scf.for %{{.*}} -> (tensor<24x25xf32>) {
   //     CHECK:   %[[PADDED:.*]] = tensor.extract_slice %[[PACKED]][%{{.*}}, 0, 0] [1, 12, 5] [1, 1, 1]
   // CHECK-SAME:    : tensor<5x12x5xf32> to tensor<12x5xf32>
-  //     CHECK:   %[[TRANSPOSED:.*]] = linalg.generic
-  //     CHECK:     -> tensor<5x12xf32>
+  //     CHECK:   %[[TRANSPOSED:.*]] = linalg.transpose ins(%[[PADDED]] : tensor<12x5xf32>)
+  //     CHECK:     outs(%{{.*}} : tensor<5x12xf32>
   //     CHECK:   linalg.matmul ins(%[[TRANSPOSED]]
   %0 = linalg.matmul ins(%arg0, %arg1 : tensor<24x12xf32>, tensor<12x25xf32>) outs(%arg2 : tensor<24x25xf32>) -> tensor<24x25xf32>
   func.return %0 : tensor<24x25xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
index 3404b73102e6a..9a43d43cd9460 100644
--- a/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization-with-patterns.mlir
@@ -1964,3 +1964,43 @@ module attributes {transform.with_named_sequence} {
 //       CHECK:     %[[VAL_8:.*]] = vector.transpose %[[VAL_7]], [1, 0] : vector<1x4xf32> to vector<4x1xf32>
 //       CHECK:     vector.transfer_write %[[VAL_8]], %{{.*}} {in_bounds = [true, true]} : vector<4x1xf32>, tensor<4x1xf32>
 //       CHECK:     vector.transfer_write %[[VAL_7]], %{{.*}} {in_bounds = [true, true]} : vector<1x4xf32>, tensor<1x4xf32>
+
+// -----
+
+// Extracted from: https://github.com/llvm/llvm-project/issues/97247
+
+#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, 0)>
+
+func.func @generic_with_reduction_and_broadcast(%arg0: tensor<1x12x197x197xf32>) -> (tensor<1x12x197x1xf32>) {
+  %0 = tensor.empty() : tensor<1x12x197x1xf32>
+  %1 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel", "reduction"]} ins(%arg0 : tensor<1x12x197x197xf32>) outs(%0 : tensor<1x12x197x1xf32>) {
+  ^bb0(%in: f32, %out: f32):
+    %818 = arith.addf %in, %out : f32
+    linalg.yield %818 : f32
+  } -> tensor<1x12x197x1xf32>
+  return %1 : tensor<1x12x197x1xf32>
+}
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// CHECK: #[[$ATTR_32:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// CHECK-LABEL:   func.func @generic_with_reduction_and_broadcast(
+// CHECK-SAME:                                                    %[[VAL_0:.*]]: tensor<1x12x197x197xf32>) -> tensor<1x12x197x1xf32> {
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = tensor.empty() : tensor<1x12x197x1xf32>
+// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]], %[[VAL_1]] {in_bounds = [true, true, true, true]} : tensor<1x12x197x197xf32>, vector<1x12x197x197xf32>
+// CHECK:           %[[VAL_5:.*]] = vector.transfer_read %[[VAL_3]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]], %[[VAL_1]] {in_bounds = [true, true, true], permutation_map = #[[$ATTR_32]]} : tensor<1x12x197x1xf32>, vector<1x12x197xf32>
+// CHECK:           %[[VAL_6:.*]] = vector.multi_reduction <add>, %[[VAL_4]], %[[VAL_5]] [3] : vector<1x12x197x197xf32> to vector<1x12x197xf32>
+// CHECK:           %[[VAL_7:.*]] = vector.broadcast %[[VAL_6]] : vector<1x12x197xf32> to vector<1x1x12x197xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transpose %[[VAL_7]], [1, 2, 3, 0] : vector<1x1x12x197xf32> to vector<1x12x197x1xf32>
+// CHECK:           %[[VAL_9:.*]] = vector.transfer_write %[[VAL_8]], %[[VAL_3]]{{\[}}%[[VAL_2]], %[[VAL_2]], %[[VAL_2]], %[[VAL_2]]] {in_bounds = [true, true, true, true]} : vector<1x12x197x1xf32>, tensor<1x12x197x1xf32>
+// CHECK:           return %[[VAL_9]] : tensor<1x12x197x1xf32>
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
index 783149971f0d6..0e2b2458d29cd 100644
--- a/mlir/test/Dialect/Linalg/vectorization.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -147,6 +147,51 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+#map = affine_map<(d0, d1) -> (d0, d1)>
+#map1 = affine_map<(d0, d1) -> (d0, 0)>
+
+func.func @dynamic_generic_with_reduction_and_broadcast(%arg0: tensor<?x?xf32>, %init: tensor<?x?xf32>) -> (tensor<?x?xf32>) {
+  %0 = linalg.generic { indexing_maps = [#map, #map1],
+                        iterator_types = ["parallel", "reduction"]}
+    ins(%arg0 : tensor<?x?xf32>)
+    outs(%init : tensor<?x?xf32>) {
+  ^bb0(%in: f32, %out: f32):
+    %1 = arith.addf %in, %out : f32
+    linalg.yield %1 : f32
+  } -> tensor<?x?xf32>
+  return %0 : tensor<?x?xf32>
+}
+// CHECK: #[[$MAP:.+]] = affine_map<(d0, d1) -> (d0)>
+
+// CHECK-LABEL:   func.func @dynamic_generic_with_reduction_and_broadcast(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<?x?xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = tensor.dim %[[VAL_0]], %[[VAL_2]] : tensor<?x?xf32>
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = tensor.dim %[[VAL_0]], %[[VAL_4]] : tensor<?x?xf32>
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_8:.*]] = vector.create_mask %[[VAL_3]], %[[VAL_5]] : vector<4x4xi1>
+// CHECK:           %[[VAL_9:.*]] = vector.mask %[[VAL_8]] { vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_7]] {in_bounds = [true, true]} : tensor<?x?xf32>, vector<4x4xf32> } : vector<4x4xi1> -> vector<4x4xf32>
+// CHECK:           %[[VAL_10:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_11:.*]] = vector.create_mask %[[VAL_3]] : vector<4xi1>
+// CHECK:           %[[VAL_12:.*]] = vector.mask %[[VAL_11]] { vector.transfer_read %[[VAL_1]]{{\[}}%[[VAL_6]], %[[VAL_6]]], %[[VAL_10]] {in_bounds = [true], permutation_map = #[[$MAP]]} : tensor<?x?xf32>, vector<4xf32> } : vector<4xi1> -> vector<4xf32>
+// CHECK:           %[[VAL_13:.*]] = vector.mask %[[VAL_8]] { vector.multi_reduction <add>, %[[VAL_9]], %[[VAL_12]] [1] : vector<4x4xf32> to vector<4xf32> } : vector<4x4xi1> -> vector<4xf32>
+// CHECK:           %[[VAL_14:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_15:.*]] = vector.mask %[[VAL_11]] { vector.transfer_write %[[VAL_13]], %[[VAL_1]]{{\[}}%[[VAL_14]], %[[VAL_14]]] {in_bounds = [true], permutation_map = #[[$MAP]]} : vector<4xf32>, tensor<?x?xf32> } : vector<4xi1> -> tensor<?x?xf32>
+// CHECK:           return %[[VAL_15]] : tensor<?x?xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %0 vector_sizes [4, 4] : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 func.func @vectorize_dynamic_2d_transpose(%arg0: tensor<?x?xf32>,
                                           %arg1: tensor<?x?xf32>,
                                           %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
diff --git a/mlir/test/Dialect/Math/extend-to-supported-types-f16.mlir b/mlir/test/Dialect/Math/extend-to-supported-types-f16.mlir
new file mode 100644
index 0000000000000..3674a91ef425f
--- /dev/null
+++ b/mlir/test/Dialect/Math/extend-to-supported-types-f16.mlir
@@ -0,0 +1,146 @@
+// RUN: mlir-opt %s --split-input-file -math-extend-to-supported-types="extra-types=f16 target-type=f32" | FileCheck %s
+
+// CHECK-LABEL: @sin_f8E5M2
+// CHECK-SAME: ([[ARG0:%.+]]: f8E5M2)
+func.func @sin_f8E5M2(%arg0: f8E5M2) -> f8E5M2 {
+  // CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]]
+  // CHECK: [[SIN:%.+]] = math.sin [[EXTF]]
+  // CHECK: [[TRUNCF:%.+]] = arith.truncf [[SIN]]
+  // CHECK: return [[TRUNCF]] : f8E5M2
+  %0 = math.sin %arg0 : f8E5M2
+  return %0 : f8E5M2
+}
+
+// CHECK-LABEL: @sin
+// CHECK-SAME: ([[ARG0:%.+]]: f16)
+func.func @sin(%arg0: f16) -> f16 {
+  // CHECK16: [[SIN:%.+]] = math.sin [[ARG0]] : f16
+  // CHECK16: return [[SIN]] : f16
+  %0 = math.sin %arg0 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: @fpowi_f8E5M2
+// CHECK-SAME: ([[ARG0:%.+]]: f8E5M2, [[ARG1:%.+]]: i32)
+func.func @fpowi_f8E5M2(%arg0: f8E5M2, %arg1: i32) -> f8E5M2 {
+  // CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]]
+  // CHECK: [[FPOWI:%.+]] = math.fpowi [[EXTF]], [[ARG1]]
+  // CHECK: [[TRUNCF:%.+]] = arith.truncf [[FPOWI]]
+  // CHECK: return [[TRUNCF]] : f8E5M2
+  %0 = math.fpowi %arg0, %arg1 : f8E5M2, i32
+  return %0 : f8E5M2
+}
+
+// CHECK-LABEL: @fpowi
+// CHECK-SAME: ([[ARG0:%.+]]: f16, [[ARG1:%.+]]: i32)
+func.func @fpowi(%arg0: f16, %arg1: i32) -> f16 {
+  // CHECK: [[FPOWI:%.+]] = math.fpowi [[ARG0]], [[ARG1]]
+  // CHECK: return [[FPOWI]] : f16
+  %0 = math.fpowi %arg0, %arg1 : f16, i32
+  return %0 : f16
+}
+
+// COM: Verify that the pass leaves `math.fma` untouched, since it is often
+// COM: implemented on small data types.
+// CHECK-LABEL: @fma
+// CHECK-SAME: ([[ARG0:%.+]]: f16, [[ARG1:%.+]]: f16, [[ARG2:%.+]]: f16)
+// CHECK: [[FMA:%.+]] = math.fma [[ARG0]], [[ARG1]], [[ARG2]]
+// CHECK: return [[FMA]] : f16
+func.func @fma(%arg0: f16, %arg1: f16, %arg2: f16) -> f16 {
+  %0 = math.fma %arg0, %arg1, %arg2 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: @absf_f16
+// CHECK-SAME: ([[ARG0:%.+]]: f16)
+// CHECK: [[ABSF:%.+]] = math.absf [[ARG0]]
+// CHECK: return [[ABSF]] : f16
+func.func @absf_f16(%arg0: f16) -> f16 {
+  %0 = math.absf %arg0 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: @absf_f32
+// CHECK-SAME: ([[ARG0:%.+]]: f32)
+// CHECK: [[ABSF:%.+]] = math.absf [[ARG0]]
+// CHECK: return [[ABSF]] : f32
+func.func @absf_f32(%arg0: f32) -> f32 {
+  %0 = math.absf %arg0 : f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: @absf_f64
+// CHECK-SAME: ([[ARG0:%.+]]: f64)
+// CHECK: [[ABSF:%.+]] = math.absf [[ARG0]]
+// CHECK: return [[ABSF]] : f64
+func.func @absf_f64(%arg0: f64) -> f64 {
+  %0 = math.absf %arg0 : f64
+  return %0 : f64
+}
+
+// CHECK-LABEL: @sin_vector
+// CHECK-SAME: ([[ARG0:%.+]]: vector<2xbf16>)
+// CHECK: [[EXTF:%.+]] = arith.extf [[ARG0]]
+// CHECK: [[SIN:%.+]] = math.sin [[EXTF]]
+// CHECK: [[TRUNCF:%.+]] = arith.truncf [[SIN]]
+// CHECK: return [[TRUNCF]] : vector<2xbf16>
+func.func @sin_vector(%arg0: vector<2xbf16>) -> vector<2xbf16> {
+  %0 = math.sin %arg0 : vector<2xbf16>
+  return %0 : vector<2xbf16>
+}
+
+// CHECK-LABEL: @sin_vector_f16
+// CHECK-SAME: ([[ARG0:%.+]]: vector<2xf16>)
+// CHECK: [[SIN:%.+]] = math.sin [[ARG0]]
+// CHECK: return [[SIN]] : vector<2xf16>
+func.func @sin_vector_f16(%arg0: vector<2xf16>) -> vector<2xf16> {
+  %0 = math.sin %arg0 : vector<2xf16>
+  return %0 : vector<2xf16>
+}
+
+// CHECK-LABEL: @fastmath
+// CHECK: math.sin %{{.+}} fastmath<nsz>
+func.func @fastmath(%arg0: f16) -> f16 {
+  %0 = math.sin %arg0 fastmath<nsz> : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: @sequences_f8E5M2
+// CHECK-SAME: ([[ARG0:%.+]]: f8E5M2)
+// CHECK: [[EXTF0:%.+]] = arith.extf [[ARG0]]
+// CHECK: [[ABSF:%.+]] = math.absf [[EXTF0]]
+// CHECK: [[TRUNCF0:%.+]] = arith.truncf [[ABSF]]
+// CHECK: [[EXTF1:%.+]] = arith.extf [[TRUNCF0]]
+// CHECK: [[SIN:%.+]] = math.sin [[EXTF1]]
+// CHECK: [[TRUNCF1:%.+]] = arith.truncf [[SIN]]
+// CHECK: return [[TRUNCF1]] : f8E5M2
+func.func @sequences_f8E5M2(%arg0: f8E5M2) -> f8E5M2 {
+  %0 = math.absf %arg0 : f8E5M2
+  %1 = math.sin %0 : f8E5M2
+  return %1 : f8E5M2
+}
+
+// CHECK-LABEL: @sequences
+// CHECK-SAME: ([[ARG0:%.+]]: f16)
+// CHECK: [[ABSF:%.+]] = math.absf [[ARG0]]
+// CHECK: [[SIN:%.+]] = math.sin [[ABSF]]
+// CHECK: return [[SIN]] : f16
+func.func @sequences(%arg0: f16) -> f16 {
+  %0 = math.absf %arg0 : f16
+  %1 = math.sin %0 : f16
+  return %1 : f16
+}
+
+// CHECK-LABEL: @promote_in_if_block
+func.func @promote_in_if_block(%arg0: bf16, %arg1: bf16, %arg2: i1) -> bf16 {
+  // CHECK: [[EXTF0:%.+]] = arith.extf
+  // CHECK-NEXT: %[[RES:.*]] = scf.if
+  %0 = scf.if %arg2 -> bf16 {
+    %1 = math.absf %arg0 : bf16
+    // CHECK: [[TRUNCF0:%.+]] = arith.truncf
+    scf.yield %1 : bf16
+  } else {
+    scf.yield %arg1 : bf16
+  }
+  return %0 : bf16
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/Math/legalize-to-f32.mlir b/mlir/test/Dialect/Math/extend-to-supported-types.mlir
similarity index 96%
rename from mlir/test/Dialect/Math/legalize-to-f32.mlir
rename to mlir/test/Dialect/Math/extend-to-supported-types.mlir
index ebb0de9d2653e..ad7169d4cf4ae 100644
--- a/mlir/test/Dialect/Math/legalize-to-f32.mlir
+++ b/mlir/test/Dialect/Math/extend-to-supported-types.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --split-input-file -math-legalize-to-f32 | FileCheck %s
+// RUN: mlir-opt %s --split-input-file -math-extend-to-supported-types="target-type=f32" | FileCheck %s
 
 // CHECK-LABEL: @sin
 // CHECK-SAME: ([[ARG0:%.+]]: f16)
diff --git a/mlir/test/Dialect/MemRef/resolve-dim-ops.mlir b/mlir/test/Dialect/MemRef/resolve-dim-ops.mlir
index 85a4853972457..ef8b80f6b5c22 100644
--- a/mlir/test/Dialect/MemRef/resolve-dim-ops.mlir
+++ b/mlir/test/Dialect/MemRef/resolve-dim-ops.mlir
@@ -71,3 +71,31 @@ func.func @dim_of_memref_expand_shape(%arg0: memref<?x8xi32>)
   %1 = memref.dim %0, %c1 : memref<1x?x2x4xi32>
   return %1 : index
 }
+
+// -----
+
+// CHECK-LABEL: @iter_to_init_arg_loop_like
+//  CHECK-SAME:   (%[[ARG0:.*]]: tensor<?x?xf32>, %[[ARG1:.*]]: tensor<?x?xf32>) -> tensor<?x?xf32> {
+//       CHECK:    %[[RESULT:.*]] = scf.forall
+//  CHECK-SAME:                       shared_outs(%[[OUTS:.*]] = %[[ARG1]]) -> (tensor<?x?xf32>) {
+//  CHECK-NEXT:       %{{.*}} = tensor.dim %[[ARG1]], %{{.*}} : tensor<?x?xf32>
+func.func @iter_to_init_arg_loop_like(
+  %arg0 : tensor<?x?xf32>, %arg1: tensor<?x?xf32>) -> tensor<?x?xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %dim0 = tensor.dim %arg0, %c0 : tensor<?x?xf32>
+
+  %result = scf.forall (%i) = (%c0) to (%dim0)
+      step (%c1) shared_outs(%o = %arg1) -> (tensor<?x?xf32>) {
+
+    %dim1 = tensor.dim %o, %c1 : tensor<?x?xf32>
+    %slice = tensor.extract_slice %arg1[%i, 0] [1, %dim1] [1, 1]
+      : tensor<?x?xf32> to tensor<1x?xf32>
+
+    scf.forall.in_parallel {
+      tensor.parallel_insert_slice %slice into %o[%i, 0] [1, %dim1] [1, 1]
+        : tensor<1x?xf32> into tensor<?x?xf32>
+    }
+  }
+  return %result : tensor<?x?xf32>
+}
diff --git a/mlir/test/Dialect/Quant/canonicalize.mlir b/mlir/test/Dialect/Quant/canonicalize.mlir
index 36c3eaf5e10d2..73c57e2a48212 100644
--- a/mlir/test/Dialect/Quant/canonicalize.mlir
+++ b/mlir/test/Dialect/Quant/canonicalize.mlir
@@ -1,24 +1,124 @@
 // RUN: mlir-opt %s -split-input-file -pass-pipeline='builtin.module(func.func(canonicalize{test-convergence}))' | FileCheck %s
 
+// CHECK-LABEL: @dcast_fold
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: return %[[ARG_0]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @dcast_fold(%arg0: tensor<4xf32>) -> tensor<4xf32> {
+  %0 = quant.qcast %arg0 : tensor<4xf32> to tensor<4x!qalias>
+  %1 = quant.dcast %0 : tensor<4x!qalias> to tensor<4xf32>
+  return %1 : tensor<4xf32>
+}
+
 // -----
-// CHECK-LABEL: redundant_scast
-func.func @redundant_scast() -> tensor<4xi8> {
-  // CHECK-NEXT: arith.constant dense<10> : tensor<4xi8>
-  // CHECK-NEXT: return
-  %cst = arith.constant dense<5> : tensor<4xi8>
-  %1 = "quant.scast"(%cst) : (tensor<4xi8>) -> tensor<4x!quant.uniform<u8:f32, 7.812500e-03:128>>
-  %2 = "quant.scast"(%1) : (tensor<4x!quant.uniform<u8:f32, 7.812500e-03:128>>) -> tensor<4xi8>
-  %3 = arith.addi %2, %2 : tensor<4xi8>
-  return %3 : tensor<4xi8>
+
+// CHECK-LABEL: @dcast_no_fold_source
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[VAL_0:.*]] = quant.scast %[[ARG_0]]
+// CHECK: %[[VAL_1:.*]] = quant.dcast %[[VAL_0]]
+// CHECK: return %[[VAL_1]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @dcast_no_fold_source(%arg0: tensor<4xi8>) -> tensor<4xf32> {
+  %0 = quant.scast %arg0 : tensor<4xi8> to tensor<4x!qalias>
+  %1 = quant.dcast %0 : tensor<4x!qalias> to tensor<4xf32>
+  return %1 : tensor<4xf32>
 }
 
 // -----
-// CHECK-LABEL: non_redundant_scast
-func.func @non_redundant_scast() -> tensor<4x!quant.uniform<u8:f32, 7.812500e-03:128>> {
-  // CHECK-NEXT: arith.constant dense<5> : tensor<4xi8>
-  // CHECK-NEXT: scast
-  // CHECK-NEXT: return
-  %cst = arith.constant dense<5> : tensor<4xi8>
-  %1 = "quant.scast"(%cst) : (tensor<4xi8>) -> tensor<4x!quant.uniform<u8:f32, 7.812500e-03:128>>
-  return %1 : tensor<4x!quant.uniform<u8:f32, 7.812500e-03:128>>
+
+// CHECK-LABEL: @qcast_fold
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: return %[[ARG_0]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @qcast_fold(%arg0: tensor<4x!qalias>) -> tensor<4x!qalias> {
+  %0 = quant.dcast %arg0 : tensor<4x!qalias> to tensor<4xf32>
+  %1 = quant.qcast %0 : tensor<4xf32> to tensor<4x!qalias>
+  return %1 : tensor<4x!qalias>
 }
+
+// -----
+
+// CHECK-LABEL: @qcast_no_fold_source
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[VAL_0:.*]] = arith.negf %[[ARG_0]]
+// CHECK: %[[VAL_1:.*]] = quant.qcast %[[VAL_0]]
+// CHECK: return %[[VAL_1]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @qcast_no_fold_source(%arg0: tensor<4xf32>) -> tensor<4x!qalias> {
+  %0 = arith.negf %arg0 : tensor<4xf32>
+  %1 = quant.qcast %0 : tensor<4xf32> to tensor<4x!qalias>
+  return %1 : tensor<4x!qalias>
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_no_fold_type
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[VAL_0:.*]] = quant.dcast %[[ARG_0]]
+// CHECK: %[[VAL_1:.*]] = quant.qcast %[[VAL_0]]
+// CHECK: return %[[VAL_1]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+!qalias1 = !quant.uniform<u8:f32, 3.0:128>
+func.func @qcast_no_fold_type(%arg0: tensor<4x!qalias>) -> tensor<4x!qalias1> {
+  %0 = quant.dcast %arg0 : tensor<4x!qalias> to tensor<4xf32>
+  %1 = quant.qcast %0 : tensor<4xf32> to tensor<4x!qalias1>
+  return %1 : tensor<4x!qalias1>
+}
+
+// -----
+
+// CHECK-LABEL: @scast_fold
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: return %[[ARG_0]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @scast_fold(%arg0: tensor<4x!qalias>) -> tensor<4x!qalias> {
+  %0 = quant.scast %arg0 : tensor<4x!qalias> to tensor<4xi8>
+  %1 = quant.scast %0 : tensor<4xi8> to tensor<4x!qalias>
+  return %1 : tensor<4x!qalias>
+}
+
+// -----
+
+// CHECK-LABEL: @scast_no_fold_source
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[QCAST:.*]] = quant.qcast %[[ARG_0]]
+// CHECK: %[[SCAST:.*]] = quant.scast %[[QCAST]]
+// CHECK: return %[[SCAST]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+func.func @scast_no_fold_source(%arg0: tensor<4xf32>) -> tensor<4xi8> {
+  %0 = quant.qcast %arg0 : tensor<4xf32> to tensor<4x!qalias>
+  %1 = quant.scast %0 : tensor<4x!qalias> to tensor<4xi8>
+  return %1 : tensor<4xi8>
+}
+
+// -----
+
+// CHECK-LABEL: @scast_no_fold_type
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[VAL_0:.*]] = quant.scast %[[ARG_0]]
+// CHECK: %[[VAL_1:.*]] = quant.scast %[[VAL_0]]
+// CHECK: return %[[VAL_1]]
+
+!qalias = !quant.uniform<u8:f32, 2.0:128>
+!qalias1 = !quant.uniform<u8:f32, 3.0:128>
+func.func @scast_no_fold_type(%arg0: tensor<4x!qalias>) -> tensor<4x!qalias1> {
+  %0 = quant.scast %arg0 : tensor<4x!qalias> to tensor<4xi8>
+  %1 = quant.scast %0 : tensor<4xi8> to tensor<4x!qalias1>
+  return %1 : tensor<4x!qalias1>
+}
+
diff --git a/mlir/test/Dialect/Quant/invalid.mlir b/mlir/test/Dialect/Quant/invalid.mlir
new file mode 100644
index 0000000000000..ba3a8e312d96e
--- /dev/null
+++ b/mlir/test/Dialect/Quant/invalid.mlir
@@ -0,0 +1,258 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+func.func @dcast_invalid_input(%arg0: f32) {
+  // expected-error@+1 {{operand #0 must be scalar or tensor of quantized type}}
+  %0 = quant.dcast %arg0 : f32 to f32
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_invalid_result(%arg0: !qalias) {
+  // expected-error@+1 {{result #0 must be scalar or tensor of floating-point}}
+  %0 = quant.dcast %arg0 : !qalias to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_mismatch_scalar_tensor(%arg0: !qalias) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.dcast %arg0 : !qalias to tensor<f32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_mismatch_ranked_unranked_tensor(%arg0: tensor<!qalias>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.dcast %arg0 : tensor<!qalias> to tensor<*xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_mismatch_static_dynamic_tensor(%arg0: tensor<2x3x!qalias>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.dcast %arg0 : tensor<2x3x!qalias> to tensor<?x3xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_float_type_mismatch(%arg0: !qalias) {
+  // expected-error@+1 {{expressed type in quantized type expected to match float type}}
+  %0 = quant.dcast %arg0 : !qalias to f64
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @dcast_per_axis_scalar(%arg0: !qalias) {
+  // expected-error@+1 {{scalar types may not use per-axis quantization}}
+  %0 = quant.dcast %arg0 : !qalias to f32
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @dcast_per_axis_invalid_rank(%arg0: tensor<2x3x!qalias>) {
+  // expected-error@+1 {{quantized dimension must be less than tensor rank}}
+  %0 = quant.dcast %arg0 : tensor<2x3x!qalias> to tensor<2x3xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @dcast_per_axis_invalid_rank(%arg0: tensor<2x3x4x!qalias>) {
+  // expected-error@+1 {{quantized dimension size does not match number of scales}}
+  %0 = quant.dcast %arg0 : tensor<2x3x4x!qalias> to tensor<2x3x4xf32>
+  return
+}
+
+// -----
+
+func.func @qcast_invalid_input(%arg0: f32) {
+  // expected-error@+1 {{result #0 must be scalar or tensor of quantized type}}
+  %0 = quant.qcast %arg0 : f32 to f32
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_invalid_result(%arg0: !qalias) {
+  // expected-error@+1 {{operand #0 must be scalar or tensor of floating-point}}
+  %0 = quant.qcast %arg0 : !qalias to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_mismatch_scalar_tensor(%arg0: tensor<f32>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.qcast %arg0 : tensor<f32> to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_mismatch_ranked_unranked_tensor(%arg0: tensor<f32>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.qcast %arg0 : tensor<f32> to tensor<*x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_mismatch_static_dynamic_tensor(%arg0: tensor<2x3xf32>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.qcast %arg0 : tensor<2x3xf32> to tensor<?x3x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_float_type_mismatch(%arg0: f64) {
+  // expected-error@+1 {{expressed type in quantized type expected to match float type}}
+  %0 = quant.qcast %arg0 : f64 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @qcast_per_axis_scalar(%arg0: f32) {
+  // expected-error@+1 {{scalar types may not use per-axis quantization}}
+  %0 = quant.qcast %arg0 : f32 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @qcast_per_axis_invalid_rank(%arg0: tensor<2x3xf32>) {
+  // expected-error@+1 {{quantized dimension must be less than tensor rank}}
+  %0 = quant.qcast %arg0 : tensor<2x3xf32> to tensor<2x3x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @qcast_per_axis_invalid_rank(%arg0: tensor<2x3x4xf32>) {
+  // expected-error@+1 {{quantized dimension size does not match number of scales}}
+  %0 = quant.qcast %arg0 : tensor<2x3x4xf32> to tensor<2x3x4x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_invalid_input(%arg0: si32) {
+  // expected-error@+1 {{operand #0 must be scalar or tensor of signless integer or quantized type}}
+  %0 = quant.scast %arg0 : si32 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_invalid_result(%arg0: !qalias) {
+  // expected-error@+1 {{result #0 must be scalar or tensor of signless integer or quantized type}}
+  %0 = quant.scast %arg0 : !qalias to si32
+  return
+}
+
+// -----
+
+func.func @scast_both_integers(%arg0: i8) {
+  // expected-error@+1 {{input must be integer and result must be quantized, or vice versa}}
+  %0 = quant.scast %arg0 : i8 to i8
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_both_quantized(%arg0: !qalias) {
+  // expected-error@+1 {{input must be integer and result must be quantized, or vice versa}}
+  %0 = quant.scast %arg0 : !qalias to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_mismatch_scalar_tensor(%arg0: tensor<i8>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.scast %arg0 : tensor<i8> to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_mismatch_ranked_unranked_tensor(%arg0: tensor<i8>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.scast %arg0 : tensor<i8> to tensor<*x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_mismatch_static_dynamic_tensor(%arg0: tensor<2x3xi8>) {
+  // expected-error@+1 {{input and result are both scalars or both tensors with matching shape}}
+  %0 = quant.scast %arg0 : tensor<2x3xi8> to tensor<?x3x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_integer_type_mismatch(%arg0: i32) {
+  // expected-error@+1 {{storage type in quantized type expected to match integer type}}
+  %0 = quant.scast %arg0 : i32 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @scast_per_axis_scalar(%arg0: i8) {
+  // expected-error@+1 {{scalar types may not use per-axis quantization}}
+  %0 = quant.scast %arg0 : i8 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0}>
+func.func @scast_per_axis_invalid_rank(%arg0: tensor<2x3xi8>) {
+  // expected-error@+1 {{quantized dimension must be less than tensor rank}}
+  %0 = quant.scast %arg0 : tensor<2x3xi8> to tensor<2x3x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @scast_per_axis_invalid_rank(%arg0: tensor<2x3x4xi8>) {
+  // expected-error@+1 {{quantized dimension size does not match number of scales}}
+  %0 = quant.scast %arg0 : tensor<2x3x4xi8> to tensor<2x3x4x!qalias>
+  return
+}
+
diff --git a/mlir/test/Dialect/Quant/lower-quant-ops.mlir b/mlir/test/Dialect/Quant/lower-quant-ops.mlir
new file mode 100644
index 0000000000000..6bba9f5c03772
--- /dev/null
+++ b/mlir/test/Dialect/Quant/lower-quant-ops.mlir
@@ -0,0 +1,511 @@
+// RUN: mlir-opt %s --lower-quant-ops --split-input-file | FileCheck %s
+
+// CHECK-LABEL: @dcast_per_layer_scalar
+// CHECK-SAME: %[[ARG_0:.*]]: !quant.uniform
+
+// CHECK: %[[STORED_INT:.*]] = quant.scast %[[ARG_0]] : !quant.uniform<i8:f32, 2.000000e+00:10> to i8
+
+// CHECK: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK: %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_INT]] : i8 to f32
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+
+// CHECK: %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK: %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE]] : f32
+// CHECK: return %[[EXPRESSED]] : f32
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @dcast_per_layer_scalar(%arg0: !qalias) -> f32 {
+  %0 = quant.dcast %arg0 : !qalias to f32
+  return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL: @dcast_per_layer_scalar_unsigned
+// CHECK-SAME: %[[ARG_0:.*]]: !quant.uniform
+
+// CHECK: %[[STORED_INT:.*]] = quant.scast %[[ARG_0]] : !quant.uniform<u8:f32, 2.000000e+00:10> to i8
+
+// CHECK: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+
+// CHECK: %[[STORED_FLOAT:.*]] = arith.uitofp %[[STORED_INT]] : i8 to f32
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.uitofp %[[ZERO_POINT]] : i8 to f32
+
+// CHECK: %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK: %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE]] : f32
+// CHECK: return %[[EXPRESSED]] : f32
+
+!qalias = !quant.uniform<u8:f32, 2.0:10>
+func.func @dcast_per_layer_scalar_unsigned(%arg0: !qalias) -> f32 {
+  %0 = quant.dcast %arg0 : !qalias to f32
+  return %0 : f32
+}
+
+// -----
+
+// CHECK-LABEL: @dcast_per_layer_0d
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[STORED_INT:.*]] = quant.scast %[[ARG_0]] : tensor<!quant.uniform<i8:f32, 2.000000e+00:10>> to tensor<i8>
+
+// CHECK: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK: %[[SCALE_TENSOR:.*]] = tensor.splat %[[SCALE]] : tensor<f32>
+// CHECK: %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_INT]] : tensor<i8> to tensor<f32>
+// CHECK: %[[ZERO_POINT_TENSOR:.*]] = tensor.splat %[[ZERO_POINT]] : tensor<i8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_TENSOR]] : tensor<i8> to tensor<f32>
+
+// CHECK: %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : tensor<f32>
+// CHECK: %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE_TENSOR]] : tensor<f32>
+// CHECK: return %[[EXPRESSED]] : tensor<f32>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @dcast_per_layer_0d(%arg0: tensor<!qalias>) -> tensor<f32> {
+  %0 = quant.dcast %arg0 : tensor<!qalias> to tensor<f32>
+  return %0 : tensor<f32>
+}
+
+// -----
+
+// CHECK-LABEL: @dcast_per_layer_ranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[STORED_INT:.*]] = quant.scast %[[ARG_0]] : tensor<3x?x5x!quant.uniform<i8:f32, 2.000000e+00:10>> to tensor<3x?x5xi8>
+// CHECK: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK: %[[C_1:.*]] = arith.constant 1 : index
+// CHECK: %[[DIM_1:.*]] = tensor.dim %[[STORED_INT]], %[[C_1]] : tensor<3x?x5xi8>
+// CHECK: %[[SCALE_TENSOR:.*]] = tensor.splat %[[SCALE]]{{\[}}%[[DIM_1]]] : tensor<3x?x5xf32>
+// CHECK: %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_INT]] : tensor<3x?x5xi8> to tensor<3x?x5xf32>
+// CHECK: %[[ZERO_POINT_TENSOR:.*]] = tensor.splat %[[ZERO_POINT]]{{\[}}%[[DIM_1]]] : tensor<3x?x5xi8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_TENSOR]] : tensor<3x?x5xi8> to tensor<3x?x5xf32>
+
+// CHECK: %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : tensor<3x?x5xf32>
+// CHECK: %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE_TENSOR]] : tensor<3x?x5xf32>
+// CHECK: return %[[EXPRESSED]] : tensor<3x?x5xf32>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @dcast_per_layer_ranked(%arg0: tensor<3x?x5x!qalias>) -> tensor<3x?x5xf32> {
+  %0 = quant.dcast %arg0 : tensor<3x?x5x!qalias> to tensor<3x?x5xf32>
+  return %0 : tensor<3x?x5xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @dcast_per_layer_unranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[STORED_INT:.*]] = quant.scast %[[ARG_0]] : tensor<*x!quant.uniform<i8:f32, 2.000000e+00:10>> to tensor<*xi8>
+// CHECK: %[[INPUT_SHAPE:.*]] = shape.shape_of %[[STORED_INT]] : tensor<*xi8> -> tensor<?xindex>
+// CHECK: %[[INPUT_SIZE:.*]] = shape.num_elements %[[INPUT_SHAPE]] : tensor<?xindex> -> index
+// CHECK: %[[COLLAPSED_SHAPE:.*]] = tensor.from_elements %[[INPUT_SIZE]] : tensor<1xindex>
+// CHECK: %[[STORED_COLLAPSED:.*]] = tensor.reshape %[[STORED_INT]](%[[COLLAPSED_SHAPE]]) : (tensor<*xi8>, tensor<1xindex>) -> tensor<?xi8>
+// CHECK: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK: %[[C_0:.*]] = arith.constant 0 : index
+// CHECK: %[[DIM_0:.*]] = tensor.dim %[[STORED_COLLAPSED]], %[[C_0]] : tensor<?xi8>
+// CHECK: %[[SCALE_TENSOR:.*]] = tensor.splat %[[SCALE]]{{\[}}%[[DIM_0]]] : tensor<?xf32>
+// CHECK: %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_COLLAPSED]] : tensor<?xi8> to tensor<?xf32>
+// CHECK: %[[ZERO_POINT_TENSOR:.*]] = tensor.splat %[[ZERO_POINT]]{{\[}}%[[DIM_0]]] : tensor<?xi8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_TENSOR]] : tensor<?xi8> to tensor<?xf32>
+
+// CHECK: %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : tensor<?xf32>
+// CHECK: %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE_TENSOR]] : tensor<?xf32>
+
+// CHECK: %[[EXPRESSED_EXPANDED:.*]] = tensor.reshape %[[EXPRESSED]](%[[INPUT_SHAPE]]) : (tensor<?xf32>, tensor<?xindex>) -> tensor<*xf32>
+// CHECK: return %[[EXPRESSED_EXPANDED]] : tensor<*xf32>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @dcast_per_layer_unranked(%arg0: tensor<*x!qalias>) -> tensor<*xf32> {
+  %0 = quant.dcast %arg0 : tensor<*x!qalias> to tensor<*xf32>
+  return %0 : tensor<*xf32>
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d1)>
+
+// CHECK-LABEL: @dcast_per_channel_ranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[STORED_TENSOR:.*]] = quant.scast %[[ARG_0]] : tensor<4x?x?x5x!quant.uniform<i8:f32:1, {2.000000e+00:10,3.000000e+00:20}>> to tensor<4x?x?x5xi8>
+
+// CHECK: %[[SCALES:.*]] = arith.constant dense<[2.000000e+00, 3.000000e+00]> : tensor<2xf32>
+// CHECK: %[[ZERO_POINTS:.*]] = arith.constant dense<[10, 20]> : tensor<2xi8>
+// CHECK: %[[C_1:.*]] = arith.constant 1 : index
+// CHECK: %[[DIM_1:.*]] = tensor.dim %[[STORED_TENSOR]], %[[C_1]] : tensor<4x?x?x5xi8>
+// CHECK: %[[C_2:.*]] = arith.constant 2 : index
+// CHECK: %[[DIM_2:.*]] = tensor.dim %[[STORED_TENSOR]], %[[C_2]] : tensor<4x?x?x5xi8>
+// CHECK: %[[INIT:.*]] = tensor.empty(%[[DIM_1]], %[[DIM_2]]) : tensor<4x?x?x5xf32>
+// CHECK: %[[GENERIC:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_1]], #[[$ATTR_0]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[STORED_TENSOR]], %[[SCALES]], %[[ZERO_POINTS]] : tensor<4x?x?x5xi8>, tensor<2xf32>, tensor<2xi8>) outs(%[[INIT]] : tensor<4x?x?x5xf32>) {
+// CHECK: ^bb0(%[[STORED_INT:.*]]: i8, %[[SCALE:.*]]: f32, %[[ZERO_POINT:.*]]: i8, %[[OUT:.*]]: f32):
+// CHECK:   %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_INT]] : i8 to f32
+// CHECK:   %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK:   %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK:   %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE]] : f32
+// CHECK:   linalg.yield %[[EXPRESSED]] : f32
+// CHECK: } -> tensor<4x?x?x5xf32>
+// CHECK: return %[[GENERIC]] : tensor<4x?x?x5xf32>
+
+!qalias = !quant.uniform<i8:f32:1, {2.0:10, 3.0:20}>
+func.func @dcast_per_channel_ranked(%arg0: tensor<4x?x?x5x!qalias>) -> tensor<4x?x?x5xf32> {
+  %0 = quant.dcast %arg0 : tensor<4x?x?x5x!qalias> to tensor<4x?x?x5xf32>
+  return %0 : tensor<4x?x?x5xf32>
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1)>
+
+// CHECK-LABEL: @dcast_per_channel_unranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor
+
+// CHECK: %[[STORED_TENSOR:.*]] = quant.scast %[[ARG_0]] : tensor<*x!quant.uniform<i8:f32:2, {2.000000e+00:10,3.000000e+00:20,4.000000e+00:30}>> to tensor<*xi8>
+// CHECK: %[[SHAPE:.*]] = shape.shape_of %[[STORED_TENSOR]] : tensor<*xi8> -> tensor<?xindex>
+// CHECK: %[[CHANNEL_AXIS:.*]] = arith.constant 2 : index
+// CHECK: %[[CHANNEL_AXIS_NEXT:.*]] = arith.constant 3 : index
+// CHECK: %[[SHAPE_LEFT:.*]], %[[DISCARDED_0:.*]] = "shape.split_at"(%[[SHAPE]], %[[CHANNEL_AXIS]]) : (tensor<?xindex>, index) -> (tensor<?xindex>, tensor<?xindex>)
+// CHECK: %[[SIZE_LEFT:.*]] = shape.num_elements %[[SHAPE_LEFT]] : tensor<?xindex> -> index
+// CHECK: %[[DISCARDED_1:.*]], %[[SHAPE_RIGHT:.*]] = "shape.split_at"(%[[SHAPE]], %[[CHANNEL_AXIS_NEXT]]) : (tensor<?xindex>, index) -> (tensor<?xindex>, tensor<?xindex>)
+// CHECK: %[[SIZE_RIGHT:.*]] = shape.num_elements %[[SHAPE_RIGHT]] : tensor<?xindex> -> index
+
+// CHECK: %[[NUM_CHANNELS:.*]] = arith.constant 3 : index
+// CHECK: %[[COLLAPSED_SHAPE:.*]] = tensor.from_elements %[[SIZE_LEFT]], %[[NUM_CHANNELS]], %[[SIZE_RIGHT]] : tensor<3xindex>
+// CHECK: %[[STORED_COLLAPSED:.*]] = tensor.reshape %[[STORED_TENSOR]](%[[COLLAPSED_SHAPE]]) : (tensor<*xi8>, tensor<3xindex>) -> tensor<?x3x?xi8>
+
+// CHECK: %[[SCALES:.*]] = arith.constant dense<[2.000000e+00, 3.000000e+00, 4.000000e+00]> : tensor<3xf32>
+// CHECK: %[[ZERO_POINTS:.*]] = arith.constant dense<[10, 20, 30]> : tensor<3xi8>
+// CHECK: %[[C_0:.*]] = arith.constant 0 : index
+// CHECK: %[[DIM_0:.*]] = tensor.dim %[[STORED_COLLAPSED]], %[[C_0]] : tensor<?x3x?xi8>
+// CHECK: %[[C_2:.*]] = arith.constant 2 : index
+// CHECK: %[[DIM_2:.*]] = tensor.dim %[[STORED_COLLAPSED]], %[[C_2]] : tensor<?x3x?xi8>
+// CHECK: %[[INIT:.*]] = tensor.empty(%[[DIM_0]], %[[DIM_2]]) : tensor<?x3x?xf32>
+// CHECK: %[[GENERIC:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_1]], #[[$ATTR_0]]], iterator_types = ["parallel", "parallel", "parallel"]} ins(%[[STORED_COLLAPSED]], %[[SCALES]], %[[ZERO_POINTS]] : tensor<?x3x?xi8>, tensor<3xf32>, tensor<3xi8>) outs(%[[INIT]] : tensor<?x3x?xf32>) {
+// CHECK: ^bb0(%[[STORED_INT:.*]]: i8, %[[SCALE:.*]]: f32, %[[ZERO_POINT:.*]]: i8, %[[OUT:.*]]: f32):
+// CHECK:   %[[STORED_FLOAT:.*]] = arith.sitofp %[[STORED_INT]] : i8 to f32
+// CHECK:   %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK:   %[[SCALED:.*]] = arith.subf %[[STORED_FLOAT]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK:   %[[EXPRESSED:.*]] = arith.mulf %[[SCALED]], %[[SCALE]] : f32
+// CHECK:   linalg.yield %[[EXPRESSED]] : f32
+// CHECK: } -> tensor<?x3x?xf32>
+
+// CHECK: %[[EXPRESSED_EXPANDED:.*]] = tensor.reshape %[[GENERIC]](%[[SHAPE]]) : (tensor<?x3x?xf32>, tensor<?xindex>) -> tensor<*xf32>
+// CHECK: return %[[EXPRESSED_EXPANDED]] : tensor<*xf32>
+
+!qalias = !quant.uniform<i8:f32:2, {2.0:10, 3.0:20, 4.0:30}>
+func.func @dcast_per_channel_unranked(%arg0: tensor<*x!qalias>) -> tensor<*xf32> {
+  %0 = quant.dcast %arg0 : tensor<*x!qalias> to tensor<*xf32>
+  return %0 : tensor<*xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_scalar
+// CHECK-SAME: %[[ARG_0:.*]]: f32
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE]] : f32
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK: %[[STORED:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[STORED]] : f32 to i8
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_INT]] : i8 to !quant.uniform<i8:f32, 2.000000e+00:10>
+// CHECK: return %[[STORED_QUANT]] : !quant.uniform<i8:f32, 2.000000e+00:10>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @qcast_per_layer_scalar(%arg0: f32) -> !qalias {
+  %0 = quant.qcast %arg0 : f32 to !qalias
+  return %0 : !qalias
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_scalar_bounds
+// CHECK-SAME: %[[ARG_0:.*]]: f32
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 0 : i8
+
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE]] : f32
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[SCALED]] : f32 to i8
+
+// CHECK-DAG: %[[C_NEG_5:.*]] = arith.constant -5 : i8
+// CHECK-DAG: %[[C_10:.*]] = arith.constant 10 : i8
+// CHECK: %[[STORED_CLAMPED_TEMP:.*]] = arith.maxsi %[[STORED_INT]], %[[C_NEG_5]] : i8
+// CHECK: %[[STORED_CLAMPED:.*]] = arith.minsi %[[STORED_CLAMPED_TEMP]], %[[C_10]] : i8
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_CLAMPED]] : i8 to !quant.uniform<i8<-5:10>:f32, 2.000000e+00>
+// CHECK: return %[[STORED_QUANT]] : !quant.uniform<i8<-5:10>:f32, 2.000000e+00>
+
+!qalias = !quant.uniform<i8<-5:10>:f32, 2.0>
+func.func @qcast_per_layer_scalar_bounds(%arg0: f32) -> !qalias {
+  %0 = quant.qcast %arg0 : f32 to !qalias
+  return %0 : !qalias
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_scalar_unsigned_bounds
+// CHECK-SAME: %[[ARG_0:.*]]: f32
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 0 : i8
+
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE]] : f32
+// CHECK: %[[STORED_INT:.*]] = arith.fptoui %[[SCALED]] : f32 to i8
+
+// CHECK-DAG: %[[C_2:.*]] = arith.constant 2 : i8
+// CHECK-DAG: %[[C_10:.*]] = arith.constant 10 : i8
+// CHECK: %[[STORED_CLAMPED_TEMP:.*]] = arith.maxui %[[STORED_INT]], %[[C_2]] : i8
+// CHECK: %[[STORED_CLAMPED:.*]] = arith.minui %[[STORED_CLAMPED_TEMP]], %[[C_10]] : i8
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_CLAMPED]] : i8 to !quant.uniform<u8<2:10>:f32, 2.000000e+00>
+// CHECK: return %[[STORED_QUANT]] : !quant.uniform<u8<2:10>:f32, 2.000000e+00>
+
+!qalias = !quant.uniform<u8<2:10>:f32, 2.0>
+func.func @qcast_per_layer_scalar_unsigned_bounds(%arg0: f32) -> !qalias {
+  %0 = quant.qcast %arg0 : f32 to !qalias
+  return %0 : !qalias
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_0d
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<f32>
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+
+// CHECK: %[[SCALE_TENSOR:.*]] = tensor.splat %[[SCALE]] : tensor<f32>
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE_TENSOR]] : tensor<f32>
+
+// CHECK: %[[ZERO_POINT_TENSOR:.*]] = tensor.splat %[[ZERO_POINT]] : tensor<i8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_TENSOR]] : tensor<i8> to tensor<f32>
+// CHECK: %[[STORED_FLOAT:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : tensor<f32>
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[STORED_FLOAT]] : tensor<f32> to tensor<i8>
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_INT]] : tensor<i8> to tensor<!quant.uniform<i8:f32, 2.000000e+00:10>>
+// CHECK: return %[[STORED_QUANT]] : tensor<!quant.uniform<i8:f32, 2.000000e+00:10>>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @qcast_per_layer_0d(%arg0: tensor<f32>) -> tensor<!qalias> {
+  %0 = quant.qcast %arg0 : tensor<f32> to tensor<!qalias>
+  return %0 : tensor<!qalias>
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_ranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<3x?x5xf32>
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK-DAG: %[[C_1:.*]] = arith.constant 1 : index
+
+// CHECK: %[[DIM_1:.*]] = tensor.dim %[[ARG_0]], %[[C_1]] : tensor<3x?x5xf32>
+// CHECK: %[[SCALE_TENSOR:.*]] = tensor.splat %[[SCALE]]{{\[}}%[[DIM_1]]] : tensor<3x?x5xf32>
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE_TENSOR]] : tensor<3x?x5xf32>
+
+// CHECK: %[[ZERO_POINT_TENSOR:.*]] = tensor.splat %[[ZERO_POINT]]{{\[}}%[[DIM_1]]] : tensor<3x?x5xi8>
+// CHECK: %[[ZERO_POINT_TENSOR_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_TENSOR]] : tensor<3x?x5xi8> to tensor<3x?x5xf32>
+// CHECK: %[[STORED:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_TENSOR_FLOAT]] : tensor<3x?x5xf32>
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[STORED]] : tensor<3x?x5xf32> to tensor<3x?x5xi8>
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_INT]] : tensor<3x?x5xi8> to tensor<3x?x5x!quant.uniform<i8:f32, 2.000000e+00:10>>
+// CHECK: return %[[STORED_QUANT]] : tensor<3x?x5x!quant.uniform<i8:f32, 2.000000e+00:10>>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @qcast_per_layer_ranked(%arg0: tensor<3x?x5xf32>) -> tensor<3x?x5x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<3x?x5xf32> to tensor<3x?x5x!qalias>
+  return %0 : tensor<3x?x5x!qalias>
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_ranked_bounds
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<3x5xf32>
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+
+// CHECK: %[[SCALE_SPLAT:.*]] = tensor.splat %[[SCALE]] : tensor<3x5xf32>
+// CHECK: %[[SCALED:.*]] = arith.divf %[[ARG_0]], %[[SCALE_SPLAT]] : tensor<3x5xf32>
+
+// CHECK: %[[ZERO_POINT_SPLAT:.*]] = tensor.splat %[[ZERO_POINT]] : tensor<3x5xi8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_SPLAT]] : tensor<3x5xi8> to tensor<3x5xf32>
+
+// CHECK: %[[STORED_FLOAT:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : tensor<3x5xf32>
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[STORED_FLOAT]] : tensor<3x5xf32> to tensor<3x5xi8>
+
+// CHECK-DAG: %[[C_NEG_8:.*]] = arith.constant -8 : i8
+// CHECK-DAG: %[[C_7:.*]] = arith.constant 7 : i8
+// CHECK-DAG: %[[SPLAT_NEG_8:.*]] = tensor.splat %[[C_NEG_8]] : tensor<3x5xi8>
+// CHECK-DAG: %[[SPLAT_7:.*]] = tensor.splat %[[C_7]] : tensor<3x5xi8>
+// CHECK: %[[STORED_CLAMPED_TEMP:.*]] = arith.maxsi %[[STORED_INT]], %[[SPLAT_NEG_8]] : tensor<3x5xi8>
+// CHECK: %[[STORED_CLAMPED:.*]] = arith.minsi %[[STORED_CLAMPED_TEMP]], %[[SPLAT_7]] : tensor<3x5xi8>
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_CLAMPED]] : tensor<3x5xi8> to tensor<3x5x!quant.uniform<i8<-8:7>:f32, 2.000000e+00:10>>
+// CHECK: return %[[STORED_QUANT]] : tensor<3x5x!quant.uniform<i8<-8:7>:f32, 2.000000e+00:10>>
+
+!qalias = !quant.uniform<i8<-8:7>:f32, 2.0:10>
+func.func @qcast_per_layer_ranked_bounds(%arg0: tensor<3x5xf32>) -> tensor<3x5x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<3x5xf32> to tensor<3x5x!qalias>
+  return %0 : tensor<3x5x!qalias>
+}
+
+// -----
+
+// CHECK-LABEL: @qcast_per_layer_unranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<*xf32>
+
+// CHECK: %[[SHAPE:.*]] = shape.shape_of %[[ARG_0]] : tensor<*xf32> -> tensor<?xindex>
+// CHECK: %[[SIZE:.*]] = shape.num_elements %[[SHAPE]] : tensor<?xindex> -> index
+// CHECK: %[[SIZE_TENSOR:.*]] = tensor.from_elements %[[SIZE]] : tensor<1xindex>
+// CHECK: %[[RANKED_INPUT:.*]] = tensor.reshape %[[ARG_0]](%[[SIZE_TENSOR]]) : (tensor<*xf32>, tensor<1xindex>) -> tensor<?xf32>
+
+// CHECK-DAG: %[[SCALE:.*]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG: %[[ZERO_POINT:.*]] = arith.constant 10 : i8
+// CHECK-DAG: %[[C_0:.*]] = arith.constant 0 : index
+
+// CHECK: %[[DIM_0:.*]] = tensor.dim %[[RANKED_INPUT]], %[[C_0]] : tensor<?xf32>
+// CHECK: %[[SCALE_SPLAT:.*]] = tensor.splat %[[SCALE]]{{\[}}%[[DIM_0]]] : tensor<?xf32>
+// CHECK: %[[SCALED:.*]] = arith.divf %[[RANKED_INPUT]], %[[SCALE_SPLAT]] : tensor<?xf32>
+
+// CHECK: %[[ZERO_POINT_SPLAT:.*]] = tensor.splat %[[ZERO_POINT]]{{\[}}%[[DIM_0]]] : tensor<?xi8>
+// CHECK: %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT_SPLAT]] : tensor<?xi8> to tensor<?xf32>
+// CHECK: %[[STORED:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : tensor<?xf32>
+// CHECK: %[[STORED_INT:.*]] = arith.fptosi %[[STORED]] : tensor<?xf32> to tensor<?xi8>
+
+// CHECK: %[[STORED_UNRANKED:.*]] = tensor.reshape %[[STORED_INT]](%[[SHAPE]]) : (tensor<?xi8>, tensor<?xindex>) -> tensor<*xi8>
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_UNRANKED]] : tensor<*xi8> to tensor<*x!quant.uniform<i8:f32, 2.000000e+00:10>>
+// CHECK: return %[[STORED_QUANT]] : tensor<*x!quant.uniform<i8:f32, 2.000000e+00:10>>
+
+!qalias = !quant.uniform<i8:f32, 2.0:10>
+func.func @qcast_per_layer_unranked(%arg0: tensor<*xf32>) -> tensor<*x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<*xf32> to tensor<*x!qalias>
+  return %0 : tensor<*x!qalias>
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d1)>
+
+// CHECK-LABEL: @qcast_per_channel_ranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<4x?x?x5xf32>
+
+// CHECK: %[[SCALES:.*]] = arith.constant dense<[2.000000e+00, 3.000000e+00]> : tensor<2xf32>
+// CHECK: %[[ZERO_POINTS:.*]] = arith.constant dense<[10, 20]> : tensor<2xi8>
+
+// CHECK-DAG: %[[C_1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[DIM_1:.*]] = tensor.dim %[[ARG_0]], %[[C_1]] : tensor<4x?x?x5xf32>
+// CHECK-DAG: %[[C_2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[DIM_2:.*]] = tensor.dim %[[ARG_0]], %[[C_2]] : tensor<4x?x?x5xf32>
+// CHECK: %[[INIT:.*]] = tensor.empty(%[[DIM_1]], %[[DIM_2]]) : tensor<4x?x?x5xi8>
+
+// CHECK: %[[GENERIC:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_1]], #[[$ATTR_0]]], iterator_types = ["parallel", "parallel", "parallel", "parallel"]} ins(%[[ARG_0]], %[[SCALES]], %[[ZERO_POINTS]] : tensor<4x?x?x5xf32>, tensor<2xf32>, tensor<2xi8>) outs(%[[INIT]] : tensor<4x?x?x5xi8>) {
+// CHECK: ^bb0(%[[IN:.*]]: f32, %[[SCALE:.*]]: f32, %[[ZERO_POINT:.*]]: i8, %[[OUT:.*]]: i8):
+// CHECK:   %[[SCALED:.*]] = arith.divf %[[IN]], %[[SCALE]] : f32
+// CHECK:   %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK:   %[[STORED_FLOAT:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK:   %[[STORED_INT:.*]] = arith.fptosi %[[STORED_FLOAT]] : f32 to i8
+// CHECK:   linalg.yield %[[STORED_INT]] : i8
+// CHECK: } -> tensor<4x?x?x5xi8>
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[GENERIC]] : tensor<4x?x?x5xi8> to tensor<4x?x?x5x!quant.uniform<i8:f32:1, {2.000000e+00:10,3.000000e+00:20}>>
+// CHECK: return %[[STORED_QUANT]] : tensor<4x?x?x5x!quant.uniform<i8:f32:1, {2.000000e+00:10,3.000000e+00:20}>>
+
+!qalias = !quant.uniform<i8:f32:1, {2.0:10, 3.0:20}>
+func.func @qcast_per_channel_ranked(%arg0: tensor<4x?x?x5xf32>) -> tensor<4x?x?x5x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<4x?x?x5xf32> to tensor<4x?x?x5x!qalias>
+  return %0 : tensor<4x?x?x5x!qalias>
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1)>
+
+// CHECK-LABEL: @qcast_per_channel_ranked_bounds
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<4x2x5xf32>
+
+// CHECK: %[[SCALES:.*]] = arith.constant dense<[2.000000e+00, 3.000000e+00]> : tensor<2xf32>
+// CHECK: %[[ZERO_POINTS:.*]] = arith.constant dense<0> : tensor<2xi8>
+
+// CHECK: %[[INIT:.*]] = tensor.empty() : tensor<4x2x5xi8>
+// CHECK: %[[GENERIC:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_1]], #[[$ATTR_0]]], iterator_types = ["parallel", "parallel", "parallel"]} ins(%[[ARG_0]], %[[SCALES]], %[[ZERO_POINTS]] : tensor<4x2x5xf32>, tensor<2xf32>, tensor<2xi8>) outs(%[[INIT]] : tensor<4x2x5xi8>) {
+// CHECK: ^bb0(%[[IN:.*]]: f32, %[[SCALE:.*]]: f32, %[[ZERO_POINT:.*]]: i8, %[[OUT:.*]]: i8):
+// CHECK:   %[[SCALED:.*]] = arith.divf %[[IN]], %[[SCALE]] : f32
+// CHECK:   %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK:   %[[STORED_FLOAT:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK:   %[[STORED_INT:.*]] = arith.fptosi %[[STORED_FLOAT]] : f32 to i8
+// CHECK:   %[[C_NEG_8:.*]] = arith.constant -8 : i8
+// CHECK:   %[[C_7:.*]] = arith.constant 7 : i8
+// CHECK:   %[[STORED_CLAMPED_TEMP:.*]] = arith.maxsi %[[STORED_INT]], %[[C_NEG_8]] : i8
+// CHECK:   %[[STORED_CLAMPED:.*]] = arith.minsi %[[STORED_CLAMPED_TEMP]], %[[C_7]] : i8
+// CHECK:   linalg.yield %[[STORED_CLAMPED]] : i8
+// CHECK: } -> tensor<4x2x5xi8>
+
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[GENERIC]] : tensor<4x2x5xi8> to tensor<4x2x5x!quant.uniform<i8<-8:7>:f32:1, {2.000000e+00,3.000000e+00}>>
+// CHECK: return %[[STORED_QUANT]] : tensor<4x2x5x!quant.uniform<i8<-8:7>:f32:1, {2.000000e+00,3.000000e+00}>>
+
+!qalias = !quant.uniform<i8<-8:7>:f32:1, {2.0, 3.0}>
+func.func @qcast_per_channel_ranked_bounds(%arg0: tensor<4x2x5xf32>) -> tensor<4x2x5x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<4x2x5xf32> to tensor<4x2x5x!qalias>
+  return %0 : tensor<4x2x5x!qalias>
+}
+
+// -----
+
+// CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
+// CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2) -> (d1)>
+
+// CHECK-LABEL: @qcast_per_channel_unranked
+// CHECK-SAME: %[[ARG_0:.*]]: tensor<*xf32>
+
+// CHECK: %[[SHAPE:.*]] = shape.shape_of %[[ARG_0]] : tensor<*xf32> -> tensor<?xindex>
+// CHECK: %[[CHANNEL_AXIS:.*]] = arith.constant 2 : index
+// CHECK: %[[CHANNEL_AXIS_NEXT:.*]] = arith.constant 3 : index
+// CHECK: %[[SHAPE_LEFT:.*]], %[[DISCARDED_0:.*]] = "shape.split_at"(%[[SHAPE]], %[[CHANNEL_AXIS]]) : (tensor<?xindex>, index) -> (tensor<?xindex>, tensor<?xindex>)
+// CHECK: %[[SIZE_LEFT:.*]] = shape.num_elements %[[SHAPE_LEFT]] : tensor<?xindex> -> index
+// CHECK: %[[DISCARDED_1:.*]], %[[SHAPE_RIGHT:.*]] = "shape.split_at"(%[[SHAPE]], %[[CHANNEL_AXIS_NEXT]]) : (tensor<?xindex>, index) -> (tensor<?xindex>, tensor<?xindex>)
+// CHECK: %[[SIZE_RIGHT:.*]] = shape.num_elements %[[SHAPE_RIGHT]] : tensor<?xindex> -> index
+
+// CHECK: %[[CHANNEL_AXIS_SIZE:.*]] = arith.constant 3 : index
+// CHECK: %[[FLAT_SHAPE:.*]] = tensor.from_elements %[[SIZE_LEFT]], %[[CHANNEL_AXIS_SIZE]], %[[SIZE_RIGHT]] : tensor<3xindex>
+// CHECK: %[[FLAT_INPUT:.*]] = tensor.reshape %[[ARG_0]](%[[FLAT_SHAPE]]) : (tensor<*xf32>, tensor<3xindex>) -> tensor<?x3x?xf32>
+
+// CHECK: %[[SCALES:.*]] = arith.constant dense<[2.000000e+00, 3.000000e+00, 4.000000e+00]> : tensor<3xf32>
+// CHECK: %[[ZERO_POINTS:.*]] = arith.constant dense<[10, 20, 30]> : tensor<3xi8>
+
+// CHECK: %[[C_0:.*]] = arith.constant 0 : index
+// CHECK: %[[DIM_0:.*]] = tensor.dim %[[FLAT_INPUT]], %[[C_0]] : tensor<?x3x?xf32>
+// CHECK: %[[C_2:.*]] = arith.constant 2 : index
+// CHECK: %[[DIM_2:.*]] = tensor.dim %[[FLAT_INPUT]], %[[C_2]] : tensor<?x3x?xf32>
+// CHECK: %[[INIT:.*]] = tensor.empty(%[[DIM_0]], %[[DIM_2]]) : tensor<?x3x?xi8>
+
+// CHECK: %[[GENERIC:.*]] = linalg.generic {indexing_maps = [#[[$ATTR_0]], #[[$ATTR_1]], #[[$ATTR_1]], #[[$ATTR_0]]], iterator_types = ["parallel", "parallel", "parallel"]} ins(%[[FLAT_INPUT]], %[[SCALES]], %[[ZERO_POINTS]] : tensor<?x3x?xf32>, tensor<3xf32>, tensor<3xi8>) outs(%[[INIT]] : tensor<?x3x?xi8>) {
+// CHECK: ^bb0(%[[IN:.*]]: f32, %[[SCALE:.*]]: f32, %[[ZERO_POINT:.*]]: i8, %[[OUT:.*]]: i8):
+// CHECK:   %[[SCALED:.*]] = arith.divf %[[IN]], %[[SCALE]] : f32
+// CHECK:   %[[ZERO_POINT_FLOAT:.*]] = arith.sitofp %[[ZERO_POINT]] : i8 to f32
+// CHECK:   %[[STORED_FLOAT:.*]] = arith.addf %[[SCALED]], %[[ZERO_POINT_FLOAT]] : f32
+// CHECK:   %[[STORED_INT:.*]] = arith.fptosi %[[STORED_FLOAT]] : f32 to i8
+// CHECK:   linalg.yield %[[STORED_INT]] : i8
+// CHECK: } -> tensor<?x3x?xi8>
+
+// CHECK: %[[STORED_UNRANKED:.*]] = tensor.reshape %[[GENERIC]](%[[SHAPE]]) : (tensor<?x3x?xi8>, tensor<?xindex>) -> tensor<*xi8>
+// CHECK: %[[STORED_QUANT:.*]] = quant.scast %[[STORED_UNRANKED]] : tensor<*xi8> to tensor<*x!quant.uniform<i8:f32:2, {2.000000e+00:10,3.000000e+00:20,4.000000e+00:30}>>
+// CHECK: return %[[STORED_QUANT]] : tensor<*x!quant.uniform<i8:f32:2, {2.000000e+00:10,3.000000e+00:20,4.000000e+00:30}>>
+
+!qalias = !quant.uniform<i8:f32:2, {2.0:10, 3.0:20, 4.0:30}>
+func.func @qcast_per_channel_unranked(%arg0: tensor<*xf32>) -> tensor<*x!qalias> {
+  %0 = quant.qcast %arg0 : tensor<*xf32> to tensor<*x!qalias>
+  return %0 : tensor<*x!qalias>
+}
+
diff --git a/mlir/test/Dialect/Quant/ops.mlir b/mlir/test/Dialect/Quant/ops.mlir
new file mode 100644
index 0000000000000..4abc5830d081e
--- /dev/null
+++ b/mlir/test/Dialect/Quant/ops.mlir
@@ -0,0 +1,151 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_scalar(%arg0: !qalias) {
+  %0 = quant.dcast %arg0 : !qalias to f32
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_ranked(%arg0: tensor<2x?x4x!qalias>) {
+  %0 = quant.dcast %arg0 : tensor<2x?x4x!qalias> to tensor<2x?x4xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @dcast_unranked(%arg0: tensor<*x!qalias>) {
+  %0 = quant.dcast %arg0 : tensor<*x!qalias> to tensor<*xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @dcast_per_axis_static(%arg0: tensor<1x2x3x!qalias>) {
+  %0 = quant.dcast %arg0 : tensor<1x2x3x!qalias> to tensor<1x2x3xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @dcast_per_axis_dynamic(%arg0: tensor<?x?x?x!qalias>) {
+  %0 = quant.dcast %arg0 : tensor<?x?x?x!qalias> to tensor<?x?x?xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @dcast_per_axis_unranked(%arg0: tensor<*x!qalias>) {
+  %0 = quant.dcast %arg0 : tensor<*x!qalias> to tensor<*xf32>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_scalar(%arg0: f32) {
+  %0 = quant.qcast %arg0 : f32 to !qalias
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_ranked(%arg0: tensor<2x?x4xf32>) {
+  %0 = quant.qcast %arg0 : tensor<2x?x4xf32> to tensor<2x?x4x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @qcast_unranked(%arg0: tensor<*xf32>) {
+  %0 = quant.qcast %arg0 : tensor<*xf32> to tensor<*x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @qcast_per_axis_static(%arg0: tensor<1x2x3xf32>) {
+  %0 = quant.qcast %arg0 : tensor<1x2x3xf32> to tensor<1x2x3x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @qcast_per_axis_dynamic(%arg0: tensor<?x?x?xf32>) {
+  %0 = quant.qcast %arg0 : tensor<?x?x?xf32> to tensor<?x?x?x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @qcast_per_axis_unranked(%arg0: tensor<*xf32>) {
+  %0 = quant.qcast %arg0 : tensor<*xf32> to tensor<*x!qalias>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_scalar(%arg0: i8) {
+  %0 = quant.scast %arg0 : i8 to !qalias
+  %1 = quant.scast %0 : !qalias to i8
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_ranked(%arg0: tensor<2x?x4xi8>) {
+  %0 = quant.scast %arg0 : tensor<2x?x4xi8> to tensor<2x?x4x!qalias>
+  %1 = quant.scast %0 : tensor<2x?x4x!qalias> to tensor<2x?x4xi8>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32, 1.0>
+func.func @scast_unranked(%arg0: tensor<*xi8>) {
+  %0 = quant.scast %arg0 : tensor<*xi8> to tensor<*x!qalias>
+  %1 = quant.scast %0 : tensor<*x!qalias> to tensor<*xi8>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @scast_per_axis_static(%arg0: tensor<1x2x3xi8>) {
+  %0 = quant.scast %arg0 : tensor<1x2x3xi8> to tensor<1x2x3x!qalias>
+  %1 = quant.scast %0 : tensor<1x2x3x!qalias> to tensor<1x2x3xi8>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @scast_per_axis_dynamic(%arg0: tensor<?x?x?xi8>) {
+  %0 = quant.scast %arg0 : tensor<?x?x?xi8> to tensor<?x?x?x!qalias>
+  %1 = quant.scast %0 : tensor<?x?x?x!qalias> to tensor<?x?x?xi8>
+  return
+}
+
+// -----
+
+!qalias = !quant.uniform<i8:f32:2, {1.0, 2.0, 3.0}>
+func.func @scast_per_axis_unranked(%arg0: tensor<*xi8>) {
+  %0 = quant.scast %arg0 : tensor<*xi8> to tensor<*x!qalias>
+  %1 = quant.scast %0 : tensor<*x!qalias> to tensor<*xi8>
+  return
+}
+
+
diff --git a/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir b/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
index a82e8efdb1a3c..7613a344cf2b8 100644
--- a/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
+++ b/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
@@ -120,3 +120,28 @@
 // provided.
 // expected-error@+1 {{expected floating point literal}}
 !qalias = !quant.uniform<i8<-4:3>:f32, {2.000000e+02,-19.987200e-01:1}>
+
+// -----
+// Illegal negative axis in per-axis quantization
+// expected-error@+1 {{illegal quantized dimension: -1}}
+!qalias = !quant.uniform<i8:f32:-1, {2.0,3.0:1}>
+
+// -----
+// Scale f16 underflow
+// expected-error@+1 {{scale out of expressed type range}}
+!qalias = !quant.uniform<i8:f16, 5.8e-8>
+
+// -----
+// Scale f16 overflow
+// expected-error@+1 {{scale out of expressed type range}}
+!qalias = !quant.uniform<i8:f16, 6.6e4>
+
+// -----
+// Scale f16 underflow in per-axis quantization
+// expected-error@+1 {{scale out of expressed type range}}
+!qalias = !quant.uniform<i8:f16:1, {2.0,5.8e-8}>
+
+// -----
+// Scale f16 overflow in per-axis quantization
+// expected-error@+1 {{scale out of expressed type range}}
+!qalias = !quant.uniform<i8:f16:1, {2.0,6.6e4}>
diff --git a/mlir/test/Dialect/Quant/strip-func-quant-types.mlir b/mlir/test/Dialect/Quant/strip-func-quant-types.mlir
new file mode 100644
index 0000000000000..e5f0d4921bed3
--- /dev/null
+++ b/mlir/test/Dialect/Quant/strip-func-quant-types.mlir
@@ -0,0 +1,88 @@
+// RUN: mlir-opt %s --strip-func-quant-types --split-input-file | FileCheck %s
+
+// CHECK-LABEL: @strip_operands
+// CHECK-SAME: %[[ARG_0:.*]]: i8
+// CHECK-SAME: %[[ARG_1:.*]]: i16
+// CHECK-SAME: %[[ARG_2:.*]]: f32
+
+// CHECK: %[[ARG_0_CAST:.*]] = quant.scast %[[ARG_1]] : i16 to !quant.uniform<{{.*}}>
+// CHECK: %[[ARG_1_CAST:.*]] = quant.scast %[[ARG_0]] : i8 to !quant.uniform<{{.*}}>
+
+// CHECK: "test.custom_op"(%[[ARG_1_CAST]])
+// CHECK: "test.custom_op"(%[[ARG_0_CAST]])
+// CHECK: "test.custom_op"(%[[ARG_2]])
+
+!qalias = !quant.uniform<i8:f32, 2.0:128>
+!qalias1 = !quant.uniform<i16:f32, 3.0:128>
+
+func.func @strip_operands(%arg0: !qalias, %arg1: !qalias1, %arg2: f32) {
+  "test.custom_op"(%arg0) : (!qalias) -> tensor<4x!qalias>
+  "test.custom_op"(%arg1) : (!qalias1) -> tensor<?x!qalias1>
+  "test.custom_op"(%arg2) : (f32) -> tensor<4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @strip_results
+// CHECK-SAME: tensor<4xi8>, tensor<?xi16>, tensor<*xi8>, tensor<4xf32>
+
+// CHECK: %[[RESULT_0:.*]] = "test.custom_op"()
+// CHECK: %[[RESULT_CAST_0:.*]] = quant.scast %[[RESULT_0]] : tensor<4x!quant.uniform<{{.*}}>> to tensor<4xi8>
+
+// CHECK: %[[RESULT_1:.*]] = "test.custom_op"()
+// CHECK: %[[RESULT_CAST_1:.*]] = quant.scast %[[RESULT_1]] : tensor<?x!quant.uniform<{{.*}}>> to tensor<?xi16>
+
+// CHECK: %[[RESULT_2:.*]] = "test.custom_op"()
+// CHECK: %[[RESULT_CAST_2:.*]] = quant.scast %[[RESULT_2]] : tensor<*x!quant.uniform<{{.*}}>> to tensor<*xi8>
+
+// CHECK: %[[RESULT_3:.*]] = "test.custom_op"()
+
+// CHECK: return %[[RESULT_CAST_0]], %[[RESULT_CAST_1]], %[[RESULT_CAST_2]], %[[RESULT_3]]
+
+!qalias = !quant.uniform<i8:f32, 2.0:128>
+!qalias1 = !quant.uniform<i16:f32, 3.0:128>
+
+func.func @strip_results() -> (tensor<4x!qalias>, tensor<?x!qalias1>, tensor<*x!qalias>, tensor<4xf32>) {
+  %0 = "test.custom_op"() : () -> tensor<4x!qalias>
+  %1 = "test.custom_op"() : () -> tensor<?x!qalias1>
+  %2 = "test.custom_op"() : () -> tensor<*x!qalias>
+  %3 = "test.custom_op"() : () -> tensor<4xf32>
+  return %0, %1, %2, %3 : tensor<4x!qalias>, tensor<?x!qalias1>, tensor<*x!qalias>, tensor<4xf32>
+}
+
+// -----
+
+
+// CHECK-LABEL: @callee
+// CHECK-SAME: (tensor<4xi8>, tensor<?xi16>) -> (tensor<*xi8>, tensor<4xf32>)
+
+// CHECK-LABEL: @strip_call
+
+// CHECK: %[[OPERAND_0:.*]] = "test.custom_op"()
+// CHECK: %[[OPERAND_0_CAST:.*]] = quant.scast %[[OPERAND_0]] : tensor<4x!quant.uniform<{{.*}}>> to tensor<4xi8>
+
+// CHECK: %[[OPERAND_1:.*]] = "test.custom_op"()
+// CHECK: %[[OPERAND_1_CAST:.*]] = quant.scast %[[OPERAND_1]] : tensor<?x!quant.uniform<{{.*}}>> to tensor<?xi16>
+
+// CHECK: %[[RESULTS:.*]]:2 = call @callee(%[[OPERAND_0_CAST]], %[[OPERAND_1_CAST]])
+
+// CHECK: %[[RESULT_0_CAST:.*]] = quant.scast %[[RESULTS]]#0 : tensor<*xi8> to tensor<*x!quant.uniform<{{.*}}>>
+// CHECK: "test.custom_op"(%[[RESULT_0_CAST]])
+
+// CHECK: "test.custom_op"(%[[RESULTS]]#1)
+
+// CHECK: return
+
+!qalias = !quant.uniform<i8:f32, 2.0:128>
+!qalias1 = !quant.uniform<i16:f32, 3.0:128>
+
+func.func private @callee(tensor<4x!qalias>, tensor<?x!qalias1>) -> (tensor<*x!qalias>, tensor<4xf32>)
+
+func.func @strip_call() {
+  %0 = "test.custom_op"() : () -> tensor<4x!qalias>
+  %1 = "test.custom_op"() : () -> tensor<?x!qalias1>
+  %2:2 = func.call @callee(%0, %1) : (tensor<4x!qalias>, tensor<?x!qalias1>) -> (tensor<*x!qalias>, tensor<4xf32>)
+  "test.custom_op"(%2#0) : (tensor<*x!qalias>) -> ()
+  "test.custom_op"(%2#1) : (tensor<4xf32>) -> ()
+  return
+}
diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir
index 4747aad977a49..af49d2afc049b 100644
--- a/mlir/test/Dialect/SCF/loop-pipelining.mlir
+++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir
@@ -766,8 +766,11 @@ func.func @stage_0_value_escape(%A: memref<?xf32>, %result: memref<?xf32>, %ub:
 
 // Check for predicated epilogue for dynamic loop.
 // CHECK-LABEL: dynamic_loop(
-//        CHECK:   %[[C0:.*]] = arith.constant 0 : index
-//        CHECK:   %{{.*}}:2 = scf.for %[[ARG5:.*]] = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})
+//    CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+//    CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//    CHECK-DAG:   %[[CM1:.*]] = arith.constant -1 : index
+//        CHECK:   %[[UBM:.*]] = arith.subi %[[UB:.*]], %{{.*}}
+//        CHECK:   %{{.*}}:2 = scf.for %[[ARG5:.*]] = %[[LB:.*]] to %[[UBM]] step %[[STEP:.*]] iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})
 //        CHECK:       memref.store %[[ARG6]], %{{.*}}[%[[ARG5]]]
 //        CHECK:       %[[ADDF_24:.*]] = arith.addf %[[ARG7]], %{{.*}}
 //        CHECK:       %[[MULI_25:.*]] = arith.muli %{{.*}}, %{{.*}}
@@ -775,15 +778,17 @@ func.func @stage_0_value_escape(%A: memref<?xf32>, %result: memref<?xf32>, %ub:
 //        CHECK:       %[[LOAD_27:.*]] = memref.load %{{.*}}[%[[ADDI_26]]]
 //        CHECK:       scf.yield %[[ADDF_24]], %[[LOAD_27]]
 //        CHECK:   }
-//        CHECK:   %[[SUBI_10:.*]] = arith.subi %{{.*}}, %{{.*}}
-//        CHECK:   %[[ADDI_11:.*]] = arith.addi %[[SUBI_10]], %{{.*}}
-//        CHECK:   %[[ADDI_12:.*]] = arith.addi %[[ADDI_11]], %{{.*}}-1
-//        CHECK:   %[[DIVUI_13:.*]] = arith.divui %[[ADDI_12]], %{{.*}}
-//        CHECK:   %[[ADDI_14:.*]] = arith.addi %[[DIVUI_13]], %{{.*}}-1
+//        CHECK:   %[[CMPI_10:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]]
+//        CHECK:   %[[SEL_10:.*]] = arith.select %[[CMPI_10]], %[[C1]], %[[CM1]]
+//        CHECK:   %[[SUBI_10:.*]] = arith.subi %[[UB]], %[[LB]]
+//        CHECK:   %[[ADDI_11:.*]] = arith.addi %[[SUBI_10]], %[[STEP]]
+//        CHECK:   %[[ADDI_12:.*]] = arith.addi %[[ADDI_11]], %[[SEL_10]]
+//        CHECK:   %[[DIVSI_13:.*]] = arith.divsi %[[ADDI_12]], %[[STEP]]
+//        CHECK:   %[[ADDI_14:.*]] = arith.addi %[[DIVSI_13]], %[[CM1]]
 //        CHECK:   %[[MULI_15:.*]] = arith.muli %{{.*}}, %[[ADDI_14]]
 //        CHECK:   %[[ADDI_16:.*]] = arith.addi %{{.*}}, %[[MULI_15]]
 //        CHECK:   %[[CMPI_17:.*]] = arith.cmpi sge, %[[ADDI_14]], %[[C0]]
-//        CHECK:   %[[ADDI_18:.*]] = arith.addi %[[DIVUI_13]], %{{.*}}-1
+//        CHECK:   %[[ADDI_18:.*]] = arith.addi %[[DIVSI_13]], %{{.*}}-1
 //        CHECK:   %[[ADDI_19:.*]] = arith.addi %[[ADDI_18]], %{{.*}}-1
 //        CHECK:   %[[MULI_20:.*]] = arith.muli %{{.*}}, %[[ADDI_19]]
 //        CHECK:   %[[ADDI_21:.*]] = arith.addi %{{.*}}, %[[MULI_20]]
@@ -834,32 +839,38 @@ func.func @dynamic_loop(%A: memref<?xf32>, %result: memref<?xf32>, %lb: index, %
 
 // Check for predicated epilogue for dynamic loop.
 // CHECK-LABEL:   func.func @dynamic_loop_result
-//       CHECK:     %{{.*}}:2 = scf.for %[[ARG5:.*]] = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})
+//   CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[CM1:.*]] = arith.constant -1 : index
+//       CHECK:   %[[UBM:.*]] = arith.subi %[[UB:.*]], %{{.*}}
+//       CHECK:   %{{.*}}:2 = scf.for %[[ARG5:.*]] = %[[LB:.*]] to %[[UBM]] step %[[STEP:.*]] iter_args(%[[ARG6:.*]] = %{{.*}}, %[[ARG7:.*]] = %{{.*}})
 //       CHECK:       %[[ADDF_13:.*]] = arith.addf %[[ARG7]], %[[ARG6]]
 //       CHECK:       %[[MULF_14:.*]] = arith.mulf %[[ADDF_13]], %{{.*}}
 //       CHECK:       %[[ADDI_15:.*]] = arith.addi %[[ARG5]], %{{.*}}
 //       CHECK:       %[[LOAD_16:.*]] = memref.load %{{.*}}[%[[ADDI_15]]]
 //       CHECK:       scf.yield %[[MULF_14]], %[[LOAD_16]]
 //       CHECK:     }
-//       CHECK:     %[[SUBI_4:.*]] = arith.subi %{{.*}}, %{{.*}}
-//       CHECK:     %[[ADDI_5:.*]] = arith.addi %[[SUBI_4]], %{{.*}}
-//       CHECK:     %[[ADDI_6:.*]] = arith.addi %[[ADDI_5]], %{{.*}}-1
-//       CHECK:     %[[DIVUI_7:.*]] = arith.divui %[[ADDI_6]], %{{.*}}
-//       CHECK:     %[[ADDI_8:.*]] = arith.addi %[[DIVUI_7]], %{{.*}}-1
-//       CHECK:     %[[CMPI_9:.*]] = arith.cmpi sge, %[[ADDI_8]], %{{.*}}
-//       CHECK:     %[[IF_10:.*]] = scf.if %[[CMPI_9]]
+//       CHECK:     %[[CMPI_4:.*]] = arith.cmpi slt, %[[STEP]], %[[C0]]
+//       CHECK:     %[[SELECT_5:.*]] = arith.select %[[CMPI_4]], %[[C1]], %[[CM1]]
+//       CHECK:     %[[SUBI_6:.*]] = arith.subi %[[UB]], %[[LB]]
+//       CHECK:     %[[ADDI_7:.*]] = arith.addi %[[SUBI_6]], %[[STEP]]
+//       CHECK:     %[[ADDI_8:.*]] = arith.addi %[[ADDI_7]], %[[SELECT_5]]
+//       CHECK:     %[[DIVSI_9:.*]] = arith.divsi %[[ADDI_8]], %[[STEP]]
+//       CHECK:     %[[ADDI_10:.*]] = arith.addi %[[DIVSI_9]], %[[CM1]]
+//       CHECK:     %[[CMPI_11:.*]] = arith.cmpi sge, %[[ADDI_10]], %[[C0]]
+//       CHECK:     %[[IF_10:.*]] = scf.if %[[CMPI_11]]
 //       CHECK:       %[[ADDF_13:.*]] = arith.addf %{{.*}}#1, %{{.*}}#0
 //       CHECK:       scf.yield %[[ADDF_13]]
 //       CHECK:     } else {
 //       CHECK:       scf.yield %{{.*}}
 //       CHECK:     }
-//       CHECK:     %[[IF_11:.*]] = scf.if %[[CMPI_9]]
+//       CHECK:     %[[IF_11:.*]] = scf.if %[[CMPI_11]]
 //       CHECK:       %[[MULF_13:.*]] = arith.mulf %[[IF_10]], %{{.*}}
 //       CHECK:       scf.yield %[[MULF_13]]
 //       CHECK:     } else {
 //       CHECK:       scf.yield %{{.*}}
 //       CHECK:     }
-//       CHECK:     %[[SELECT_12:.*]] = arith.select %[[CMPI_9]], %[[IF_11]], %{{.*}}#0
+//       CHECK:     %[[SELECT_12:.*]] = arith.select %[[CMPI_11]], %[[IF_11]], %{{.*}}#0
 //       CHECK:     memref.store %[[SELECT_12]], %{{.*}}[%{{.*}}]
 func.func @dynamic_loop_result(%A: memref<?xf32>, %result: memref<?xf32>, %lb: index, %ub: index, %step: index) {
   %cf0 = arith.constant 1.0 : f32
diff --git a/mlir/test/Dialect/Vector/drop-unit-dims-with-shape-cast.mlir b/mlir/test/Dialect/Vector/drop-unit-dims-with-shape-cast.mlir
index af3fc924c1dbe..34a155fbf2fc1 100644
--- a/mlir/test/Dialect/Vector/drop-unit-dims-with-shape-cast.mlir
+++ b/mlir/test/Dialect/Vector/drop-unit-dims-with-shape-cast.mlir
@@ -207,3 +207,100 @@ func.func @negative_transpose_with_no_unit_dims(%vec: vector<4x2x3xf32>) -> vect
 
 // CHECK-LABEL: func.func @negative_transpose_with_no_unit_dims
 // CHECK-NOT: vector.shape_cast
+
+// -----
+
+///----------------------------------------------------------------------------------------
+/// [Pattern: DropUnitDimsFromScfForOp]
+///----------------------------------------------------------------------------------------
+
+func.func @scf_for_with_internal_unit_dims(%vec: vector<4x1x1x[4]xf32>) -> vector<4x1x1x[4]xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %res = scf.for %i = %c0 to %c4 step %c1 iter_args(%iter = %vec) -> vector<4x1x1x[4]xf32> {
+    %s = math.sqrt %iter : vector<4x1x1x[4]xf32>
+    scf.yield %s : vector<4x1x1x[4]xf32>
+  }
+  return %res : vector<4x1x1x[4]xf32>
+}
+
+// CHECK-LABEL: func.func @scf_for_with_internal_unit_dims
+//  CHECK-SAME:   %[[VEC:[A-Za-z0-9]+]]: vector<4x1x1x[4]xf32>
+//       CHECK:   %[[CAST:.+]] = vector.shape_cast %[[VEC]] : vector<4x1x1x[4]xf32> to vector<4x[4]xf32>
+//       CHECK:   %[[LOOP:.+]] = scf.for {{.*}} iter_args(%[[ITER:.+]] = %[[CAST]])
+//       CHECK:     %[[SQRT:.+]] = math.sqrt %[[ITER]] : vector<4x[4]xf32>
+//       CHECK:     scf.yield %[[SQRT]]
+//       CHECK:   %[[CASTBACK:.+]] = vector.shape_cast %[[LOOP]] : vector<4x[4]xf32> to vector<4x1x1x[4]xf32>
+//       CHECK:   return %[[CASTBACK]]
+
+// -----
+
+func.func @scf_for_with_all_unit_dims(%vec: vector<1x1xf32>) -> vector<1x1xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %res = scf.for %i = %c0 to %c4 step %c1 iter_args(%iter = %vec) -> vector<1x1xf32> {
+    %s = math.sqrt %iter : vector<1x1xf32>
+    scf.yield %s : vector<1x1xf32>
+  }
+  return %res : vector<1x1xf32>
+}
+
+// CHECK-LABEL: func.func @scf_for_with_all_unit_dims
+//  CHECK-SAME:   %[[VEC:[A-Za-z0-9]+]]: vector<1x1xf32>
+//       CHECK:   %[[CAST:.+]] = vector.shape_cast %[[VEC]] : vector<1x1xf32> to vector<1xf32>
+//       CHECK:   %[[LOOP:.+]] = scf.for {{.*}} iter_args(%[[ITER:.+]] = %[[CAST]])
+//       CHECK:     %[[SQRT:.+]] = math.sqrt %[[ITER]] : vector<1xf32>
+//       CHECK:     scf.yield %[[SQRT]]
+//       CHECK:   %[[CASTBACK:.+]] = vector.shape_cast %[[LOOP]] : vector<1xf32> to vector<1x1xf32>
+//       CHECK:   return %[[CASTBACK]]
+
+// -----
+
+func.func @scf_for_with_multiple_operands(%idx: index, %vec0: vector<1x4xf32>, %vec1: vector<1x4xf32>) -> vector<1x4xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %res:3 = scf.for %i = %c0 to %c4 step %c1
+    iter_args(%id = %idx, %iter0 = %vec0, %iter1 = %vec1) -> (index, vector<1x4xf32>, vector<1x4xf32>) {
+    %add = arith.addf %iter0, %iter1 : vector<1x4xf32>
+    scf.yield %id, %add, %add : index, vector<1x4xf32>, vector<1x4xf32>
+  }
+  return %res#1 : vector<1x4xf32>
+}
+
+// CHECK-LABEL: func.func @scf_for_with_multiple_operands
+//  CHECK-SAME:   %[[IDX:[A-Za-z0-9]+]]: index
+//  CHECK-SAME:   %[[VEC0:[A-Za-z0-9]+]]: vector<1x4xf32>
+//  CHECK-SAME:   %[[VEC1:[A-Za-z0-9]+]]: vector<1x4xf32>
+//   CHECK-DAG:   %[[CAST0:.+]] = vector.shape_cast %[[VEC0]] : vector<1x4xf32> to vector<4xf32>
+//   CHECK-DAG:   %[[CAST1:.+]] = vector.shape_cast %[[VEC1]] : vector<1x4xf32> to vector<4xf32>
+//       CHECK:   %[[LOOP:.+]]:3 = scf.for
+//  CHECK-SAME:     iter_args(%{{.*}} = %[[IDX]], %[[ITER0:.+]] = %[[CAST0]], %[[ITER1:.+]] = %[[CAST1]])
+//       CHECK:     %[[ADD:.+]] = arith.addf %[[ITER0]], %[[ITER1]] : vector<4xf32>
+//       CHECK:     scf.yield %{{.*}}, %[[ADD]], %[[ADD]]
+//       CHECK:   %[[CASTBACK:.+]] = vector.shape_cast %[[LOOP]]#1 : vector<4xf32> to vector<1x4xf32>
+//       CHECK:   return %[[CASTBACK]]
+
+// -----
+
+func.func @scf_for_with_scalable_unit_dims(%vec: vector<1x[1]xf32>) -> vector<1x[1]xf32> {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %res = scf.for %i = %c0 to %c4 step %c1 iter_args(%iter = %vec) -> vector<1x[1]xf32> {
+    %s = math.sqrt %iter : vector<1x[1]xf32>
+    scf.yield %s : vector<1x[1]xf32>
+  }
+  return %res : vector<1x[1]xf32>
+}
+
+// CHECK-LABEL: func.func @scf_for_with_scalable_unit_dims
+//  CHECK-SAME:   %[[VEC:[A-Za-z0-9]+]]: vector<1x[1]xf32>
+//       CHECK:   %[[CAST:.+]] = vector.shape_cast %[[VEC]] : vector<1x[1]xf32> to vector<[1]xf32>
+//       CHECK:   %[[LOOP:.+]] = scf.for {{.*}} iter_args(%[[ITER:.+]] = %[[CAST]])
+//       CHECK:     %[[SQRT:.+]] = math.sqrt %[[ITER]] : vector<[1]xf32>
+//       CHECK:     scf.yield %[[SQRT]]
+//       CHECK:   %[[CASTBACK:.+]] = vector.shape_cast %[[LOOP]] : vector<[1]xf32> to vector<1x[1]xf32>
+//       CHECK:   return %[[CASTBACK]]
diff --git a/mlir/test/Integration/GPU/CUDA/dump-sass.mlir b/mlir/test/Integration/GPU/CUDA/dump-sass.mlir
new file mode 100644
index 0000000000000..d32f5efc29d58
--- /dev/null
+++ b/mlir/test/Integration/GPU/CUDA/dump-sass.mlir
@@ -0,0 +1,18 @@
+// RUN: mlir-opt %s \
+// RUN:  | mlir-opt -gpu-lower-to-nvvm-pipeline -debug-only=dump-sass \
+// RUN:  2>&1 | FileCheck %s
+
+// CHECK: MOV
+// CHECK: STG.E 
+
+func.func @other_func(%arg0 : f32, %arg1 : memref<?xf32>) {
+  %cst = arith.constant 1 : index
+  %c0 = arith.constant 0 : index
+  %cst2 = memref.dim %arg1, %c0 : memref<?xf32>
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %cst, %grid_y = %cst, %grid_z = %cst)
+             threads(%tx, %ty, %tz) in (%block_x = %cst2, %block_y = %cst, %block_z = %cst) {
+    memref.store %arg0, %arg1[%tx] : memref<?xf32>
+    gpu.terminator
+  }
+  return
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 966a00f9e3c67..007284d0ca443 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2626,3 +2626,72 @@ llvm.func @reqd_work_group_size() attributes {reqd_work_group_size = array<i32:
 llvm.func @intel_reqd_sub_group_size() attributes {intel_reqd_sub_group_size = 32 : i32}
 
 // CHECK: ![[#INTEL_REQD_SUB_GROUP_SIZE]] = !{i32 32}
+
+// -----
+
+llvm.func @foo()
+
+llvm.func @call_with_empty_opbundle() {
+  llvm.call @foo() [] : () -> ()
+  llvm.return
+}
+
+//      CHECK: define void @call_with_empty_opbundle() {
+// CHECK-NEXT:   call void @foo()
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+
+llvm.func @call_with_empty_opbundle_operands() {
+  llvm.call @foo() ["tag"()] : () -> ()
+  llvm.return
+}
+
+//      CHECK: define void @call_with_empty_opbundle_operands() {
+// CHECK-NEXT:   call void @foo() [ "tag"() ]
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+
+llvm.func @call_with_opbundle() {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.mlir.constant(3 : i32) : i32
+  llvm.call @foo() ["tag1"(%0, %1 : i32, i32), "tag2"(%2 : i32)] : () -> ()
+  llvm.return
+}
+
+//      CHECK: define void @call_with_opbundle() {
+// CHECK-NEXT:   call void @foo() [ "tag1"(i32 1, i32 2), "tag2"(i32 3) ]
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
+
+llvm.func @__gxx_personality_v0(...) -> i32
+llvm.func @invoke_with_opbundle() attributes { personality = @__gxx_personality_v0 } {
+  %0 = llvm.mlir.constant(1 : i32) : i32
+  %1 = llvm.mlir.constant(2 : i32) : i32
+  %2 = llvm.mlir.constant(3 : i32) : i32
+  llvm.invoke @foo() to ^bb2 unwind ^bb1 ["tag1"(%0, %1 : i32, i32), "tag2"(%2 : i32)] : () -> ()
+
+^bb1:
+  %3 = llvm.landingpad cleanup : !llvm.struct<(ptr, i32)>
+  llvm.return
+
+^bb2:
+  llvm.return
+}
+
+//      CHECK: define void @invoke_with_opbundle() personality ptr @__gxx_personality_v0 {
+// CHECK-NEXT:   invoke void @foo() [ "tag1"(i32 1, i32 2), "tag2"(i32 3) ]
+// CHECK-NEXT:           to label %{{.+}} unwind label %{{.+}}
+//      CHECK: }
+
+llvm.func @call_intrin_with_opbundle(%arg0 : !llvm.ptr) {
+  %0 = llvm.mlir.constant(1 : i1) : i1
+  %1 = llvm.mlir.constant(16 : i32) : i32
+  llvm.call_intrinsic "llvm.assume"(%0) ["align"(%arg0, %1 : !llvm.ptr, i32)] : (i1) -> ()
+  llvm.return
+}
+
+//      CHECK: define void @call_intrin_with_opbundle(ptr %0) {
+// CHECK-NEXT:   call void @llvm.assume(i1 true) [ "align"(ptr %0, i32 16) ]
+// CHECK-NEXT:   ret void
+// CHECK-NEXT: }
diff --git a/offload/DeviceRTL/src/Synchronization.cpp b/offload/DeviceRTL/src/Synchronization.cpp
index d6452a5d589c5..9ea8d171cc830 100644
--- a/offload/DeviceRTL/src/Synchronization.cpp
+++ b/offload/DeviceRTL/src/Synchronization.cpp
@@ -398,6 +398,10 @@ void setLock(omp_lock_t *Lock) {
   } // wait for 0 to be the read value
 }
 
+void unsetCriticalLock(omp_lock_t *Lock) { unsetLock(Lock); }
+
+void setCriticalLock(omp_lock_t *Lock) { setLock(Lock); }
+
 #pragma omp end declare variant
 ///}
 
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index 514bb89e0b644..2f1ef3e98d817 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -88,7 +88,7 @@ config.test_flags = " -I " + config.test_source_root + \
 
 # compiler specific flags
 config.test_flags_clang = ""
-config.test_flags_flang = ""
+config.test_flags_flang = "-fopenmp-version=52"
 
 if config.omp_host_rtl_directory:
     config.test_flags = config.test_flags + " -L " + \
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index b39fb8f6795e1..de5be2b05f512 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -172,7 +172,6 @@ gentbl(
     ],
 )
 
-
 gentbl(
     name = "basic_arm_sve_typeflags_gen",
     tbl_outs = [(
@@ -996,12 +995,12 @@ cc_library(
     ],
     deps = [
         ":ast_attr_gen",
+        ":ast_bytecode_opcodes_gen",
         ":ast_comment_command_info_gen",
         ":ast_comment_html_named_character_references_gen",
         ":ast_comment_html_tags_gen",
         ":ast_comment_html_tags_properties_gen",
         ":ast_comment_nodes_gen",
-        ":ast_bytecode_opcodes_gen",
         ":ast_decl_nodes_gen",
         ":ast_properties_base_gen",
         ":ast_stmt_data_collectors_gen",
@@ -1278,6 +1277,9 @@ cc_library(
     ]),
     hdrs = glob(["include/clang/Testing/*.h"]),
     includes = ["include"],
+    textual_hdrs = [
+        "include/clang/Testing/TestLanguage.def",
+    ],
     deps = [
         ":basic",
         ":frontend",
diff --git a/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
index e8c7106b28751..010a617066c7b 100644
--- a/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
@@ -67,6 +67,7 @@ cc_test(
         "//clang:ast",
         "//clang:ast_matchers",
         "//clang:frontend",
+        "//clang:testing",
         "//clang:tooling",
         "//llvm:Support",
         "//llvm:TargetParser",
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 62f1c2a50acf7..59cc651db2b60 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -144,7 +144,7 @@ expand_template(
     out = "include/llvm/Config/abi-breaking.h",
     substitutions = {
         # Define to enable checks that alter the LLVM C++ ABI
-        "#cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS": "#define LLVM_ENABLE_ABI_BREAKING_CHECKS 0",
+        "#cmakedefine01 LLVM_ENABLE_ABI_BREAKING_CHECKS": "#define LLVM_ENABLE_ABI_BREAKING_CHECKS 1",
 
         # Define to enable reverse iteration of unordered llvm containers
         "#cmakedefine01 LLVM_ENABLE_REVERSE_ITERATION": "#define LLVM_ENABLE_REVERSE_ITERATION 0",
@@ -1457,6 +1457,7 @@ cc_library(
     copts = llvm_copts,
     textual_hdrs = ["include/llvm/SandboxIR/SandboxIRValues.def"],
     deps = [
+        ":Analysis",
         ":Core",
         ":Support",
     ],
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index 1170458664250..2c73f03dd70a3 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -632,6 +632,23 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "sandboxir_tests",
+    size = "small",
+    srcs = glob(
+        ["SandboxIR/*.cpp"],
+        allow_empty = False,
+    ),
+    deps = [
+        "//llvm:AsmParser",
+        "//llvm:Core",
+        "//llvm:SandboxIR",
+        "//llvm:Support",
+        "//third-party/unittest:gmock",
+        "//third-party/unittest:gtest",
+    ],
+)
+
 cc_test(
     name = "profile_data_tests",
     size = "small",
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index f5437245e8e13..a2f70f765eec3 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -9615,6 +9615,7 @@ cc_library(
         ":PolynomialDialect",
         ":PtrDialect",
         ":QuantOps",
+        ":QuantTransforms",
         ":ROCDLDialect",
         ":ROCDLTarget",
         ":ReconcileUnrealizedCasts",
@@ -10662,8 +10663,8 @@ cc_library(
 td_library(
     name = "QuantizationOpsTdFiles",
     srcs = [
-        "include/mlir/Dialect/Quant/QuantOps.td",
-        "include/mlir/Dialect/Quant/QuantOpsBase.td",
+        "include/mlir/Dialect/Quant/IR/QuantBase.td",
+        "include/mlir/Dialect/Quant/IR/QuantOps.td",
     ],
     includes = ["include"],
     deps = [
@@ -10678,19 +10679,19 @@ gentbl_cc_library(
     tbl_outs = [
         (
             ["-gen-op-decls"],
-            "include/mlir/Dialect/Quant/QuantOps.h.inc",
+            "include/mlir/Dialect/Quant/IR/QuantOps.h.inc",
         ),
         (
             ["-gen-op-defs"],
-            "include/mlir/Dialect/Quant/QuantOps.cpp.inc",
+            "include/mlir/Dialect/Quant/IR/QuantOps.cpp.inc",
         ),
         (
             ["-gen-dialect-decls"],
-            "include/mlir/Dialect/Quant/QuantOpsDialect.h.inc",
+            "include/mlir/Dialect/Quant/IR/QuantOpsDialect.h.inc",
         ),
         (
             ["-gen-dialect-defs"],
-            "include/mlir/Dialect/Quant/QuantOpsDialect.cpp.inc",
+            "include/mlir/Dialect/Quant/IR/QuantOpsDialect.cpp.inc",
         ),
         (
             ["-gen-op-doc"],
@@ -10698,7 +10699,7 @@ gentbl_cc_library(
         ),
     ],
     tblgen = ":mlir-tblgen",
-    td_file = "include/mlir/Dialect/Quant/QuantOps.td",
+    td_file = "include/mlir/Dialect/Quant/IR/QuantOps.td",
     deps = [":QuantizationOpsTdFiles"],
 )
 
@@ -10710,11 +10711,11 @@ gentbl_cc_library(
                 "-gen-bytecode",
                 "-bytecode-dialect=Quant",
             ],
-            "include/mlir/Dialect/Quant/QuantDialectBytecode.cpp.inc",
+            "include/mlir/Dialect/Quant/IR/QuantDialectBytecode.cpp.inc",
         ),
     ],
     tblgen = ":mlir-tblgen",
-    td_file = "include/mlir/Dialect/Quant/QuantDialectBytecode.td",
+    td_file = "include/mlir/Dialect/Quant/IR/QuantDialectBytecode.td",
     deps = [
         ":BytecodeTdFiles",
     ],
@@ -10733,10 +10734,10 @@ cc_library(
         "lib/Dialect/Quant/Utils/UniformSupport.cpp",
     ],
     hdrs = [
-        "include/mlir/Dialect/Quant/FakeQuantSupport.h",
-        "include/mlir/Dialect/Quant/QuantOps.h",
-        "include/mlir/Dialect/Quant/QuantTypes.h",
-        "include/mlir/Dialect/Quant/UniformSupport.h",
+        "include/mlir/Dialect/Quant/IR/Quant.h",
+        "include/mlir/Dialect/Quant/IR/QuantTypes.h",
+        "include/mlir/Dialect/Quant/Utils/FakeQuantSupport.h",
+        "include/mlir/Dialect/Quant/Utils/UniformSupport.h",
     ],
     includes = ["include"],
     deps = [
@@ -12370,6 +12371,7 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/Transform/IR/*.h"]),
     deps = [
         ":Analysis",
+        ":BytecodeOpInterface",
         ":CallOpInterfaces",
         ":CastInterfaces",
         ":ControlFlowInterfaces",
@@ -13162,6 +13164,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":ArithDialect",
+        ":ArithUtils",
         ":DialectUtils",
         ":IR",
         ":MathDialect",
@@ -14563,3 +14566,44 @@ gentbl_cc_library(
     td_file = "include/mlir/Dialect/LLVMIR/VCIXOps.td",
     deps = [":VCIXTdFiles"],
 )
+
+gentbl_cc_library(
+    name = "QuantPassIncGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-pass-decls",
+                "-name=Quant",
+            ],
+            "include/mlir/Dialect/Quant/Transforms/Passes.h.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Quant/Transforms/Passes.td",
+    deps = [":PassBaseTdFiles"],
+)
+
+cc_library(
+    name = "QuantTransforms",
+    srcs = glob([
+        "lib/Dialect/Quant/Transforms/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Dialect/Quant/Transforms/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":ArithDialect",
+        ":FuncDialect",
+        ":FuncTransforms",
+        ":IR",
+        ":LinalgDialect",
+        ":Pass",
+        ":QuantOps",
+        ":QuantPassIncGen",
+        ":ShapeDialect",
+        ":TensorDialect",
+        ":TransformUtils",
+        "//llvm:Support",
+    ],
+)